diff options
author | B Stack <bgstack15@gmail.com> | 2020-12-10 13:25:27 -0500 |
---|---|---|
committer | B Stack <bgstack15@gmail.com> | 2020-12-10 13:25:27 -0500 |
commit | 1f05dba5702e3a9b13e9795310200821dd1e520c (patch) | |
tree | b79ee1ade76d9cab46f282e269ba9b67ec1e1e8b /hex_zero.py | |
parent | add initial Makefile for install, uninstall (diff) | |
download | hex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.tar.gz hex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.tar.bz2 hex-zero-1f05dba5702e3a9b13e9795310200821dd1e520c.zip |
add debian/ and improve installation
Diffstat (limited to 'hex_zero.py')
-rwxr-xr-x | hex_zero.py | 552 |
1 files changed, 552 insertions, 0 deletions
diff --git a/hex_zero.py b/hex_zero.py new file mode 100755 index 0000000..81c8169 --- /dev/null +++ b/hex_zero.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# https://stackoverflow.com/a/34831345/3569534 + +""" + Copyright © 2020 Mia Herkt + Licensed under the EUPL, Version 1.2 or - as soon as approved + by the European Commission - subsequent versions of the EUPL + (the "License"); + You may not use this work except in compliance with the License. + You may obtain a copy of the license at: + + https://joinup.ec.europa.eu/software/page/eupl + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + either express or implied. + See the License for the specific language governing permissions + and limitations under the License. +""" + +from flask import Flask, abort, escape, make_response, redirect, request, send_from_directory, url_for, Response +from flask_sqlalchemy import SQLAlchemy +from flask_script import Manager, Server +from flask_migrate import Migrate, MigrateCommand +from hashlib import sha256 +from humanize import naturalsize +from magic import Magic +from mimetypes import guess_extension +import logging +from logging.config import dictConfig +import os, sys +import requests +from short_url import UrlEncoder +from validators import url as url_valid + +app = Flask(__name__) +app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + +app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite" # "postgresql://0x0@/0x0" +app.config["PREFERRED_URL_SCHEME"] = "https" # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config +app.config["MAX_CONTENT_LENGTH"] = 512 * 1024 * 1024 +app.config["MAX_URL_LENGTH"] = 4096 +app.config["FHOST_STORAGE_PATH"] = "up" +app.config["FHOST_USE_X_ACCEL_REDIRECT"] = True # expect nginx by default +app.config["USE_X_SENDFILE"] = False +app.config["FHOST_EXT_OVERRIDE"] = { + "audio/flac" : ".flac", + "image/gif" : ".gif", + "image/jpeg" : ".jpg", + "image/png" : ".png", + "image/svg+xml" : ".svg", + "video/webm" : ".webm", + "video/x-matroska" : ".mkv", + "application/octet-stream" : ".bin", + "text/plain" : ".log", + "text/plain" : ".txt", + "text/x-diff" : ".diff", +} + +# default blacklist to avoid AV mafia extortion +app.config["FHOST_MIME_BLACKLIST"] = [ + "application/x-dosexec", + "application/java-archive", + "application/java-vm" +] + +app.config["FHOST_UPLOAD_BLACKLIST"] = "tornodes.txt" + +app.config["NSFW_DETECT"] = False +app.config["NSFW_THRESHOLD"] = 0.608 + +# read config file +app.config.from_pyfile("hex-zero.conf", silent=True) + +# prepare logging to file specified by configs +# do not call app.logger before this dictConfig command! +dictConfig(app.config['WSGI_LOGGING']) + +# read html template file +with open(app.config["FHOST_FRONTPAGE"], "r") as j: + frontpagestring = j.read() + +if app.config["NSFW_DETECT"]: + from nsfw_detect import NSFWDetector + nsfw = NSFWDetector() + +try: + mimedetect = Magic(mime=True, mime_encoding=False) +except: + print("""Error: You have installed the wrong version of the 'magic' module. +Please install python-magic.""") + sys.exit(1) + +if not os.path.exists(app.config["FHOST_STORAGE_PATH"]): + os.mkdir(app.config["FHOST_STORAGE_PATH"]) + +db = SQLAlchemy(app) +migrate = Migrate(app, db) + +manager = Manager(app) +manager.add_command("db", MigrateCommand) + +su = UrlEncoder(alphabet='DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-', block_size=16) + +class URL(db.Model): + id = db.Column(db.Integer, primary_key = True) + url = db.Column(db.UnicodeText, unique = True) + + def __init__(self, url): + self.url = url + + def getname(self): + return su.enbase(self.id, 1) + + def geturl(self): + return url_for("get", path=self.getname(), _external=True) + "\n" + +class File(db.Model): + id = db.Column(db.Integer, primary_key = True) + sha256 = db.Column(db.String, unique = True) + ext = db.Column(db.UnicodeText) + mime = db.Column(db.UnicodeText) + addr = db.Column(db.UnicodeText) + removed = db.Column(db.Boolean, default=False) + nsfw_score = db.Column(db.Float) + + def __init__(self, sha256, ext, mime, addr, nsfw_score): + self.sha256 = sha256 + self.ext = ext + self.mime = mime + self.addr = addr + self.nsfw_score = nsfw_score + + def getname(self): + return u"{0}{1}".format(su.enbase(self.id, 1), self.ext) + + def geturl(self): + n = self.getname() + o = n + + if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]: + o = url_for("get", path=n, _external=True, _anchor="nsfw") + "\n" + else: + o = url_for("get", path=n, _external=True) + "\n" + return o.replace("http://" + app.config["APP_HOST_LISTEN"] + ":" + str(app.config["APP_PORT"]),app.config["APP_URL"]) + + def pprint(self): + print("url: {}".format(self.getname())) + vals = vars(self) + + for v in vals: + if not v.startswith("_sa"): + print("{}: {}".format(v, vals[v])) + +def getpath(fn): + return os.path.join(app.config["FHOST_STORAGE_PATH"], fn) + +def fhost_url(scheme=None): + return app.config["APP_URL"] + if not scheme: + return url_for(".fhost", _external=True).rstrip("/") + else: + return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/") + +def is_fhost_url(url): + return url.startswith(fhost_url()) or url.startswith(fhost_url("https")) + +def shorten(url): + # handler to convert gopher links to HTTP(S) proxy + gopher = "gopher://" + length = len(gopher) + if url[:length] == gopher: + url = "https://gopher.envs.net/{}".format(url[length:]) + + if len(url) > app.config["MAX_URL_LENGTH"]: + abort(414) + + if not url_valid(url) or is_fhost_url(url) or "\n" in url: + abort(400) + + existing = URL.query.filter_by(url=url).first() + + if existing: + return existing.geturl() + else: + u = URL(url) + db.session.add(u) + db.session.commit() + + return u.geturl() + +def in_upload_bl(addr): + if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]): + with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: + check = addr.lstrip("::ffff:") + for l in bl.readlines(): + if not l.startswith("#"): + if check == l.rstrip(): + return True + + return False + +def store_file(f, addr): + if in_upload_bl(addr): + return "Your host is blocked from uploading files.\n", 451 + + data = f.stream.read() + digest = sha256(data).hexdigest() + existing = File.query.filter_by(sha256=digest).first() + + if existing: + if existing.removed: + return legal() + + epath = getpath(existing.sha256) + + if not os.path.exists(epath): + with open(epath, "wb") as of: + of.write(data) + + if existing.nsfw_score == None: + if app.config["NSFW_DETECT"]: + existing.nsfw_score = nsfw.detect(epath) + + os.utime(epath, None) + existing.addr = addr + db.session.commit() + + return existing.geturl() + else: + guessmime = mimedetect.from_buffer(data) + + if not f.content_type or not "/" in f.content_type or f.content_type == "application/octet-stream": + mime = guessmime + else: + mime = f.content_type + + if mime in app.config["FHOST_MIME_BLACKLIST"] or guessmime in app.config["FHOST_MIME_BLACKLIST"]: + abort(415) + + if mime.startswith("text/") and not "charset" in mime: + mime += "; charset=utf-8" + + ext = os.path.splitext(f.filename)[1] + + if not ext: + gmime = mime.split(";")[0] + + if not gmime in app.config["FHOST_EXT_OVERRIDE"]: + ext = guess_extension(gmime) + else: + ext = app.config["FHOST_EXT_OVERRIDE"][gmime] + else: + ext = ext[:8] + + if not ext: + ext = ".bin" + + spath = getpath(digest) + + with open(spath, "wb") as of: + of.write(data) + + if app.config["NSFW_DETECT"]: + nsfw_score = nsfw.detect(spath) + else: + nsfw_score = None + + sf = File(digest, ext, mime, addr, nsfw_score) + db.session.add(sf) + db.session.commit() + + return sf.geturl() + +def store_url(url, addr): + # handler to convert gopher links to HTTP(S) proxy + gopher = "gopher://" + length = len(gopher) + if url[:length] == gopher: + url = "https://gopher.envs.net/{}".format(url[length:]) + + if is_fhost_url(url): + return segfault(508) + + h = { "Accept-Encoding" : "identity" } + r = requests.get(url, stream=True, verify=False, headers=h) + + try: + r.raise_for_status() + except requests.exceptions.HTTPError as e: + return str(e) + "\n" + + if "content-length" in r.headers: + l = int(r.headers["content-length"]) + + if l < app.config["MAX_CONTENT_LENGTH"]: + def urlfile(**kwargs): + return type('',(),kwargs)() + + f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="") + + return store_file(f, addr) + else: + hl = naturalsize(l, binary = True) + hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) + + return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413 + else: + return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411 + +@app.route("/<path:path>") +def get(path): + p = os.path.splitext(path) + id = su.debase(p[0]) + + if p[1]: + f = File.query.get(id) + + if f and f.ext == p[1]: + if f.removed: + return legal() + + fpath = getpath(f.sha256) + + if not os.path.exists(fpath): + abort(404) + + fsize = os.path.getsize(fpath) + + if app.config["FHOST_USE_X_ACCEL_REDIRECT"]: + response = make_response() + response.headers["Content-Type"] = f.mime + response.headers["Content-Length"] = fsize + response.headers["X-Accel-Redirect"] = "/" + fpath + return response + else: + return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) + else: + u = URL.query.get(id) + + if u: + return redirect(u.url) + + abort(404) + +@app.route("/dump_files/") +@app.route("/dump_files/<int:start>") +def dump_files(start=0): + meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https")) + + def gen(): + yield meta + + for file in File.query.order_by(File.id.asc()).offset(start): + yield file.getname() + "|" + str(file.id) + "\n" + + return Response(gen(), mimetype="text/plain") + +@app.route("/dump_urls/") +@app.route("/dump_urls/<int:start>") +def dump_urls(start=0): + meta = "#FORMAT: BEACON\n#PREFIX: {}/\n\n".format(fhost_url("https")) + + def gen(): + yield meta + + for url in URL.query.order_by(URL.id.asc()).offset(start): + if url.url.startswith("http") or url.url.startswith("https"): + bar = "|" + else: + bar = "||" + + yield url.getname() + bar + url.url + "\n" + + return Response(gen(), mimetype="text/plain") + +@app.route("/", methods=["GET", "POST"]) +def fhost(): + if request.method == "POST": + out = None + + if "file" in request.files: + if app.config["USE_HTTP_X_FORWARDED_FOR"]: + stored_ip_address = request.environ["HTTP_X_FORWARDED_FOR"] + else: + stored_ip_address = request.remote_addr + out = store_file(request.files["file"], stored_ip_address) + elif "url" in request.form: + out = store_url(request.form["url"], request.remote_addr) + elif "shorten" in request.form: + out = shorten(request.form["shorten"]) + + if not out == None: + app.logger.info("Stored file: "+str(out.strip())) + return Response(out, mimetype="text/plain") + + abort(400) + else: + fmts = list(app.config["FHOST_EXT_OVERRIDE"]) + fmts.sort() + maxsize = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) + maxsizenum, maxsizeunit = maxsize.split(" ") + maxsizenum = float(maxsizenum) + maxsizehalf = maxsizenum / 2 + + if maxsizenum.is_integer(): + maxsizenum = int(maxsizenum) + if maxsizehalf.is_integer(): + maxsizehalf = int(maxsizehalf) + + # python-interpreted variables + return frontpagestring.format(fhost_url(), + maxsize, str(maxsizehalf).rjust(27), str(maxsizenum).rjust(27), + maxsizeunit.rjust(54), + ", ".join(app.config["FHOST_MIME_BLACKLIST"]),fhost_url().split("/",2)[2], + app.config["APP_URL"], app.config["ADMIN_EMAIL"]) + +@app.route("/robots.txt") +def robots(): + return """User-agent: * +Disallow: / +""" + +def legal(): + return "451 Unavailable For Legal Reasons\n", 451 + +@app.errorhandler(400) +@app.errorhandler(404) +@app.errorhandler(414) +@app.errorhandler(415) +def segfault(e): + return "Segmentation fault\n", e.code + +@app.errorhandler(404) +def notfound(e): + return make_response("404 File not found: {0}/{1}".format(fhost_url(),request.path.lstrip("/")),404) + +@manager.command +def debug(): + app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False + app.run(debug=True, port=4562,host="0.0.0.0") + +@manager.command +def permadelete(name): + id = su.debase(name) + f = File.query.get(id) + + if f: + if os.path.exists(getpath(f.sha256)): + os.remove(getpath(f.sha256)) + f.removed = True + db.session.commit() + +@manager.command +def query(name): + id = su.debase(name) + f = File.query.get(id) + + if f: + f.pprint() + +@manager.command +def queryhash(h): + f = File.query.filter_by(sha256=h).first() + + if f: + f.pprint() + +@manager.command +def queryaddr(a, nsfw=False, removed=False): + res = File.query.filter_by(addr=a) + + if not removed: + res = res.filter(File.removed != True) + + if nsfw: + res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"]) + + for f in res: + f.pprint() + +@manager.command +def deladdr(a): + res = File.query.filter_by(addr=a).filter(File.removed != True) + + for f in res: + if os.path.exists(getpath(f.sha256)): + os.remove(getpath(f.sha256)) + f.removed = True + + db.session.commit() + +def nsfw_detect(f): + try: + open(f["path"], 'r').close() + f["nsfw_score"] = nsfw.detect(f["path"]) + return f + except: + return None + +@manager.command +def update_nsfw(): + if not app.config["NSFW_DETECT"]: + print("NSFW detection is disabled in app config") + return 1 + + from multiprocessing import Pool + import tqdm + + res = File.query.filter_by(nsfw_score=None, removed=False) + + with Pool() as p: + results = [] + work = [{ "path" : getpath(f.sha256), "id" : f.id} for f in res] + + for r in tqdm.tqdm(p.imap_unordered(nsfw_detect, work), total=len(work)): + if r: + results.append({"id": r["id"], "nsfw_score" : r["nsfw_score"]}) + + db.session.bulk_update_mappings(File, results) + db.session.commit() + + +@manager.command +def querybl(nsfw=False, removed=False): + blist = [] + if os.path.isfile(app.config["FHOST_UPLOAD_BLACKLIST"]): + with open(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: + for l in bl.readlines(): + if not l.startswith("#"): + if not ":" in l: + blist.append("::ffff:" + l.rstrip()) + else: + blist.append(l.strip()) + + res = File.query.filter(File.addr.in_(blist)) + + if not removed: + res = res.filter(File.removed != True) + + if nsfw: + res = res.filter(File.nsfw_score > app.config["NSFW_THRESHOLD"]) + + for f in res: + f.pprint() + +if __name__ == "__main__": + # allow port adjustment + manager.add_command('runserver', Server(host=app.config["APP_HOST_LISTEN"], port=app.config["APP_PORT"])) + manager.run() |