aboutsummaryrefslogtreecommitdiff
path: root/hex_zero.py
diff options
context:
space:
mode:
authorB Stack <bgstack15@gmail.com>2020-12-10 13:25:27 -0500
committerB Stack <bgstack15@gmail.com>2020-12-10 13:25:27 -0500
commit1f05dba5702e3a9b13e9795310200821dd1e520c (patch)
treeb79ee1ade76d9cab46f282e269ba9b67ec1e1e8b /hex_zero.py
parentadd initial Makefile for install, uninstall (diff)
downloadhex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.tar.gz
hex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.tar.bz2
hex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.zip
add debian/ and improve installation
Diffstat (limited to 'hex_zero.py')
-rwxr-xr-xhex_zero.py552
1 files changed, 552 insertions, 0 deletions
diff --git a/hex_zero.py b/hex_zero.py
new file mode 100755
index 0000000..81c8169
--- /dev/null
+++ b/hex_zero.py
@@ -0,0 +1,552 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# https://stackoverflow.com/a/34831345/3569534
+
+"""
+ Copyright © 2020 Mia Herkt
+ Licensed under the EUPL, Version 1.2 or - as soon as approved
+ by the European Commission - subsequent versions of the EUPL
+ (the "License");
+ You may not use this work except in compliance with the License.
+ You may obtain a copy of the license at:
+
+ https://joinup.ec.europa.eu/software/page/eupl
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ either express or implied.
+ See the License for the specific language governing permissions
+ and limitations under the License.
+"""
+
+from flask import Flask, abort, escape, make_response, redirect, request, send_from_directory, url_for, Response
+from flask_sqlalchemy import SQLAlchemy
+from flask_script import Manager, Server
+from flask_migrate import Migrate, MigrateCommand
+from hashlib import sha256
+from humanize import naturalsize
+from magic import Magic
+from mimetypes import guess_extension
+import logging
+from logging.config import dictConfig
+import os, sys
+import requests
+from short_url import UrlEncoder
+from validators import url as url_valid
+
+app = Flask(__name__)
+app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+
+app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite" # "postgresql://0x0@/0x0"
+app.config["PREFERRED_URL_SCHEME"] = "https" # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config
+app.config["MAX_CONTENT_LENGTH"] = 512 * 1024 * 1024
+app.config["MAX_URL_LENGTH"] = 4096
+app.config["FHOST_STORAGE_PATH"] = "up"
+app.config["FHOST_USE_X_ACCEL_REDIRECT"] = True # expect nginx by default
+app.config["USE_X_SENDFILE"] = False
+app.config["FHOST_EXT_OVERRIDE"] = {
+ "audio/flac" : ".flac",
+ "image/gif" : ".gif",
+ "image/jpeg" : ".jpg",
+ "image/png" : ".png",
+ "image/svg+xml" : ".svg",
+ "video/webm" : ".webm",
+ "video/x-matroska" : ".mkv",
+ "application/octet-stream" : ".bin",
+ "text/plain" : ".log",
+ "text/plain" : ".txt",
+ "text/x-diff" : ".diff",
+}
+
+# default blacklist to avoid AV mafia extortion
+app.config["FHOST_MIME_BLACKLIST"] = [
+ "application/x-dosexec",
+ "application/java-archive",
+ "application/java-vm"
+]
+
+app.config["FHOST_UPLOAD_BLACKLIST"] = "tornodes.txt"
+
+app.config["NSFW_DETECT"] = False
+app.config["NSFW_THRESHOLD"] = 0.608
+
+# read config file
+app.config.from_pyfile("hex-zero.conf", silent=True)
+
+# prepare logging to file specified by configs
+# do not call app.logger before this dictConfig command!
+dictConfig(app.config['WSGI_LOGGING'])
+
+# read html template file
+with open(app.config["FHOST_FRONTPAGE"], "r") as j:
+ frontpagestring = j.read()
+
+if app.config["NSFW_DETECT"]:
+ from nsfw_detect import NSFWDetector
+ nsfw = NSFWDetector()
+
+try:
+ mimedetect = Magic(mime=True, mime_encoding=False)
+except:
+ print("""Error: You have installed the wrong version of the 'magic' module.
+Please install python-magic.""")
+ sys.exit(1)
+
+if not os.path.exists(app.config["FHOST_STORAGE_PATH"]):
+ os.mkdir(app.config["FHOST_STORAGE_PATH"])
+
+db = SQLAlchemy(app)
+migrate = Migrate(app, db)
+
+manager = Manager(app)
+manager.add_command("db", MigrateCommand)
+
+su = UrlEncoder(alphabet='DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-', block_size=16)
+
+class URL(db.Model):
+ id = db.Column(db.Integer, primary_key = True)
+ url = db.Column(db.UnicodeText, unique = True)
+
+ def __init__(self, url):
+ self.url = url
+
+ def getname(self):
+ return su.enbase(self.id, 1)
+
+ def geturl(self):
+ return url_for("get", path=self.getname(), _external=True) + "\n"
+
+class File(db.Model):
+ id = db.Column(db.Integer, primary_key = True)
+ sha256 = db.Column(db.String, unique = True)
+ ext = db.Column(db.UnicodeText)
+ mime = db.Column(db.UnicodeText)
+ addr = db.Column(db.UnicodeText)
+ removed = db.Column(db.Boolean, default=False)
+ nsfw_score = db.Column(db.Float)
+
+ def __init__(self, sha256, ext, mime, addr, nsfw_score):
+ self.sha256 = sha256
+ self.ext = ext
+ self.mime = mime
+ self.addr = addr
+ self.nsfw_score = nsfw_score
+
+ def getname(self):
+ return u"{0}{1}".format(su.enbase(self.id, 1), self.ext)
+
+ def geturl(self):
+ n = self.getname()
+ o = n
+
+ if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]:
+ o = url_for("get", path=n, _external=True, _anchor="nsfw") + "\n"
+ else:
+ o = url_for("get", path=n, _external=True) + "\n"
+ return o.replace("http://" + app.config["APP_HOST_LISTEN"] + ":" + str(app.config["APP_PORT"]),app.config["APP_URL"])
+
+ def pprint(self):
+ print("url: {}".format(self.getname()))
+ vals = vars(self)
+
+ for v in vals:
+ if not v.startswith("_sa"):
+ print("{}: {}".format(v, vals[v]))
+
+def getpath(fn):
+ return os.path.join(app.config["FHOST_STORAGE_PATH"], fn)
+
+def fhost_url(scheme=None):
+ return app.config["APP_URL"]
+ if not scheme:
+ return url_for(".fhost", _external=True).rstrip("/")
+ else:
+ return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
+
+def is_fhost_url(url):
+ return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
+
+def shorten(url):
+ # handler to convert gopher links to HTTP(S) proxy
+ gopher = "gopher://"
+ length = len(gopher)
+ if url[:length] == gopher:
+ url = "https://gopher.envs.net/{}".format(url[length:])
+
+ if len(url) > app.config["MAX_URL_LENGTH"]:
+ abort(414)
+
+ if not url_valid(url) or is_fhost_url(url) or "\n" in url:
+ abort(400)
+
+ existing = URL.query.filter_by(url=url).first()
+
+ if existing:
+ return existing.geturl()
+ else:
+ u = URL(url)
+ db.session.add(u)
+ db.session.commit()
+
+ return u.geturl()
+
+def in_upload_bl(addr):
+ if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]):
+ with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
+ check = addr.lstrip("::ffff:")
+ for l in bl.readlines():
+ if not l.startswith("#"):
+ if check == l.rstrip():
+ return True
+
+ return False
+
+def store_file(f, addr):
+ if in_upload_bl(addr):
+ return "Your host is blocked from uploading files.\n", 451
+
+ data = f.stream.read()
+ digest = sha256(data).hexdigest()
+ existing = File.query.filter_by(sha256=digest).first()
+
+ if existing:
+ if existing.removed:
+ return legal()
+
+ epath = getpath(existing.sha256)
+
+ if not os.path.exists(epath):
+ with open(epath, "wb") as of:
+ of.write(data)
+
+ if existing.nsfw_score == None:
+ if app.config["NSFW_DETECT"]:
+ existing.nsfw_score = nsfw.detect(epath)
+
+ os.utime(epath, None)
+ existing.addr = addr
+ db.session.commit()
+
+ return existing.geturl()
+ else:
+ guessmime = mimedetect.from_buffer(data)
+
+ if not f.content_type or not "/" in f.content_type or f.content_type == "application/octet-stream":
+ mime = guessmime
+ else:
+ mime = f.content_type
+
+ if mime in app.config["FHOST_MIME_BLACKLIST"] or guessmime in app.config["FHOST_MIME_BLACKLIST"]:
+ abort(415)
+
+ if mime.startswith("text/") and not "charset" in mime:
+ mime += "; charset=utf-8"
+
+ ext = os.path.splitext(f.filename)[1]
+
+ if not ext:
+ gmime = mime.split(";")[0]
+
+ if not gmime in app.config["FHOST_EXT_OVERRIDE"]:
+ ext = guess_extension(gmime)
+ else:
+ ext = app.config["FHOST_EXT_OVERRIDE"][gmime]
+ else:
+ ext = ext[:8]
+
+ if not ext:
+ ext = ".bin"
+
+ spath = getpath(digest)
+
+ with open(spath, "wb") as of:
+ of.write(data)
+
+ if app.config["NSFW_DETECT"]:
+ nsfw_score = nsfw.detect(spath)
+ else:
+ nsfw_score = None
+
+ sf = File(digest, ext, mime, addr, nsfw_score)
+ db.session.add(sf)
+ db.session.commit()
+
+ return sf.geturl()
+
+def store_url(url, addr):
+ # handler to convert gopher links to HTTP(S) proxy
+ gopher = "gopher://"
+ length = len(gopher)
+ if url[:length] == gopher:
+ url = "https://gopher.envs.net/{}".format(url[length:])
+
+ if is_fhost_url(url):
+ return segfault(508)
+
+ h = { "Accept-Encoding" : "identity" }
+ r = requests.get(url, stream=True, verify=False, headers=h)
+
+ try:
+ r.raise_for_status()
+ except requests.exceptions.HTTPError as e:
+ return str(e) + "\n"
+
+ if "content-length" in r.headers:
+ l = int(r.headers["content-length"])
+
+ if l < app.config["MAX_CONTENT_LENGTH"]:
+ def urlfile(**kwargs):
+ return type('',(),kwargs)()
+
+ f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="")
+
+ return store_file(f, addr)
+ else:
+ hl = naturalsize(l, binary = True)
+ hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True)
+
+ return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413
+ else:
+ return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411
+
+@app.route("/<path:path>")
+def get(path):
+ p = os.path.splitext(path)
+ id = su.debase(p[0])
+
+ if p[1]:
+ f = File.query.get(id)
+
+ if f and f.ext == p[1]:
+ if f.removed:
+ return legal()
+
+ fpath = getpath(f.sha256)
+
+ if not os.path.exists(fpath):
+ abort(404)
+
+ fsize = os.path.getsize(fpath)
+
+ if app.config["FHOST_USE_X_ACCEL_REDIRECT"]:
+ response = make_response()
+ response.headers["Content-Type"] = f.mime
+ response.headers["Content-Length"] = fsize
+ response.headers["X-Accel-Redirect"] = "/" + fpath
+ return response
+ else:
+ return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
+ else:
+ u = URL.query.get(id)
+
+ if u:
+ return redirect(u.url)
+
+ abort(404)
+
+@app.route("/dump_files/")
+@app.route("/dump_files/<int:start>")
+def dump_files(start=0):
+ meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https"))
+
+ def gen():
+ yield meta
+
+ for file in File.query.order_by(File.id.asc()).offset(start):
+ yield file.getname() + "|" + str(file.id) + "\n"
+
+ return Response(gen(), mimetype="text/plain")
+
+@app.route("/dump_urls/")
+@app.route("/dump_urls/<int:start>")
+def dump_urls(start=0):
+ meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https"))
+
+ def gen():
+ yield meta
+
+ for url in URL.query.order_by(URL.id.asc()).offset(start):
+ if url.url.startswith("http") or url.url.startswith("https"):
+ bar = "|"
+ else:
+ bar = "||"
+
+ yield url.getname() + bar + url.url + "\n"
+
+ return Response(gen(), mimetype="text/plain")
+
+@app.route("/", methods=["GET", "POST"])
+def fhost():
+ if request.method == "POST":
+ out = None
+
+ if "file" in request.files:
+ if app.config["USE_HTTP_X_FORWARDED_FOR"]:
+ stored_ip_address = request.environ["HTTP_X_FORWARDED_FOR"]
+ else:
+ stored_ip_address = request.remote_addr
+ out = store_file(request.files["file"], stored_ip_address)
+ elif "url" in request.form:
+ out = store_url(request.form["url"], request.remote_addr)
+ elif "shorten" in request.form:
+ out = shorten(request.form["shorten"])
+
+ if not out == None:
+ app.logger.info("Stored file: "+str(out.strip()))
+ return Response(out, mimetype="text/plain")
+
+ abort(400)
+ else:
+ fmts = list(app.config["FHOST_EXT_OVERRIDE"])
+ fmts.sort()
+ maxsize = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True)
+ maxsizenum, maxsizeunit = maxsize.split(" ")
+ maxsizenum = float(maxsizenum)
+ maxsizehalf = maxsizenum / 2
+
+ if maxsizenum.is_integer():
+ maxsizenum = int(maxsizenum)
+ if maxsizehalf.is_integer():
+ maxsizehalf = int(maxsizehalf)
+
+ # python-interpreted variables
+ return frontpagestring.format(fhost_url(),
+ maxsize, str(maxsizehalf).rjust(27), str(maxsizenum).rjust(27),
+ maxsizeunit.rjust(54),
+ ", ".join(app.config["FHOST_MIME_BLACKLIST"]),fhost_url().split("/",2)[2],
+ app.config["APP_URL"], app.config["ADMIN_EMAIL"])
+
+@app.route("/robots.txt")
+def robots():
+ return """User-agent: *
+Disallow: /
+"""
+
+def legal():
+ return "451 Unavailable For Legal Reasons\n", 451
+
+@app.errorhandler(400)
+@app.errorhandler(404)
+@app.errorhandler(414)
+@app.errorhandler(415)
+def segfault(e):
+ return "Segmentation fault\n", e.code
+
+@app.errorhandler(404)
+def notfound(e):
+ return make_response("404 File not found: {0}/{1}".format(fhost_url(),request.path.lstrip("/")),404)
+
+@manager.command
+def debug():
+ app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False
+ app.run(debug=True, port=4562,host="0.0.0.0")
+
+@manager.command
+def permadelete(name):
+ id = su.debase(name)
+ f = File.query.get(id)
+
+ if f:
+ if os.path.exists(getpath(f.sha256)):
+ os.remove(getpath(f.sha256))
+ f.removed = True
+ db.session.commit()
+
+@manager.command
+def query(name):
+ id = su.debase(name)
+ f = File.query.get(id)
+
+ if f:
+ f.pprint()
+
+@manager.command
+def queryhash(h):
+ f = File.query.filter_by(sha256=h).first()
+
+ if f:
+ f.pprint()
+
+@manager.command
+def queryaddr(a, nsfw=False, removed=False):
+ res = File.query.filter_by(addr=a)
+
+ if not removed:
+ res = res.filter(File.removed != True)
+
+ if nsfw:
+ res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"])
+
+ for f in res:
+ f.pprint()
+
+@manager.command
+def deladdr(a):
+ res = File.query.filter_by(addr=a).filter(File.removed != True)
+
+ for f in res:
+ if os.path.exists(getpath(f.sha256)):
+ os.remove(getpath(f.sha256))
+ f.removed = True
+
+ db.session.commit()
+
+def nsfw_detect(f):
+ try:
+ open(f["path"], 'r').close()
+ f["nsfw_score"] = nsfw.detect(f["path"])
+ return f
+ except:
+ return None
+
+@manager.command
+def update_nsfw():
+ if not app.config["NSFW_DETECT"]:
+ print("NSFW detection is disabled in app config")
+ return 1
+
+ from multiprocessing import Pool
+ import tqdm
+
+ res = File.query.filter_by(nsfw_score=None, removed=False)
+
+ with Pool() as p:
+ results = []
+ work = [{ "path" : getpath(f.sha256), "id" : f.id} for f in res]
+
+ for r in tqdm.tqdm(p.imap_unordered(nsfw_detect, work), total=len(work)):
+ if r:
+ results.append({"id": r["id"], "nsfw_score" : r["nsfw_score"]})
+
+ db.session.bulk_update_mappings(File, results)
+ db.session.commit()
+
+
+@manager.command
+def querybl(nsfw=False, removed=False):
+ blist = []
+ if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]):
+ with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
+ for l in bl.readlines():
+ if not l.startswith("#"):
+ if not ":" in l:
+ blist.append("::ffff:" + l.rstrip())
+ else:
+ blist.append(l.strip())
+
+ res = File.query.filter(File.addr.in_(blist))
+
+ if not removed:
+ res = res.filter(File.removed != True)
+
+ if nsfw:
+ res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"])
+
+ for f in res:
+ f.pprint()
+
+if __name__ == "__main__":
+ # allow port adjustment
+ manager.add_command('runserver', Server(host=app.config["APP_HOST_LISTEN"], port=app.config["APP_PORT"]))
+ manager.run()
bgstack15