From 56f0295be203771f59e94162b62fa17d4fea54a3 Mon Sep 17 00:00:00 2001 From: Martin Herkt Date: Tue, 1 Nov 2016 05:17:54 +0100 Subject: init --- fhost.py | 402 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100755 fhost.py (limited to 'fhost.py') diff --git a/fhost.py b/fhost.py new file mode 100755 index 0000000..1b9a4c7 --- /dev/null +++ b/fhost.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from flask import Flask, abort, escape, make_response, redirect, request, send_from_directory, url_for +from flask_sqlalchemy import SQLAlchemy +from flask_script import Manager +from flask_migrate import Migrate, MigrateCommand +from hashlib import sha256 +from humanize import naturalsize +from magic import Magic +from mimetypes import guess_extension +import os, sys +import requests +from short_url import UrlEncoder +from validators import url as url_valid + +app = Flask(__name__) +app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + +app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///db.sqlite" # "postgresql://0x0@/0x0" +app.config["PREFERRED_URL_SCHEME"] = "https" # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config +app.config["MAX_CONTENT_LENGTH"] = 256 * 1024 * 1024 +app.config["MAX_URL_LENGTH"] = 4096 +app.config["FHOST_STORAGE_PATH"] = "up" +app.config["FHOST_USE_X_ACCEL_REDIRECT"] = True # expect nginx by default +app.config["USE_X_SENDFILE"] = False +app.config["FHOST_EXT_OVERRIDE"] = { + "image/gif" : ".gif", + "image/jpeg" : ".jpg", + "image/png" : ".png", + "image/svg+xml" : ".svg", + "video/webm" : ".webm", + "video/x-matroska" : ".mkv", + "application/octet-stream" : ".bin", + "text/plain" : ".txt" +} + +# default blacklist to avoid AV mafia extortion +app.config["FHOST_MIME_BLACKLIST"] = [ + "application/x-dosexec", + "application/java-archive", + "application/java-vm" +] + +try: + mimedetect = Magic(mime=True, mime_encoding=False) +except: + print("""Error: You have installed the wrong version of the 'magic' module. +Please install python-magic.""") + sys.exit(1) + +if not os.path.exists(app.config["FHOST_STORAGE_PATH"]): + os.mkdir(app.config["FHOST_STORAGE_PATH"]) + +db = SQLAlchemy(app) +migrate = Migrate(app, db) + +manager = Manager(app) +manager.add_command("db", MigrateCommand) + +su = UrlEncoder(alphabet='DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-', block_size=16) + +class URL(db.Model): + id = db.Column(db.Integer, primary_key = True) + url = db.Column(db.UnicodeText, unique = True) + + def __init__(self, url): + self.url = url + + def getname(self): + return su.enbase(self.id, 1) + +class File(db.Model): + id = db.Column(db.Integer, primary_key = True) + sha256 = db.Column(db.String, unique = True) + ext = db.Column(db.UnicodeText) + mime = db.Column(db.UnicodeText) + addr = db.Column(db.UnicodeText) + removed = db.Column(db.Boolean, default=False) + + def __init__(self, sha256, ext, mime, addr): + self.sha256 = sha256 + self.ext = ext + self.mime = mime + self.addr = addr + + def getname(self): + return u"{0}{1}".format(su.enbase(self.id, 1), self.ext) + + +def getpath(fn): + return os.path.join(app.config["FHOST_STORAGE_PATH"], fn) + +def geturl(p): + return url_for("get", path=p, _external=True) + "\n" + +def shorten(url): + if len(url) > app.config["MAX_URL_LENGTH"]: + abort(414) + + if not url_valid(url): + abort(400) + + existing = URL.query.filter_by(url=url).first() + + if existing: + return geturl(existing.getname()) + else: + u = URL(url) + db.session.add(u) + db.session.commit() + + return geturl(u.getname()) + +def store_file(f, addr): + data = f.stream.read() + digest = sha256(data).hexdigest() + existing = File.query.filter_by(sha256=digest).first() + + if existing: + if existing.removed: + return legal() + + epath = getpath(existing.sha256) + + if not os.path.exists(epath): + with open(epath, "wb") as of: + of.write(data) + + os.utime(epath, None) + existing.addr = addr + db.session.commit() + + return geturl(existing.getname()) + else: + guessmime = mimedetect.from_buffer(data) + + if not f.content_type or not "/" in f.content_type or f.content_type == "application/octet-stream": + mime = guessmime + else: + mime = f.content_type + + if mime in app.config["FHOST_MIME_BLACKLIST"] or guessmime in app.config["FHOST_MIME_BLACKLIST"]: + abort(415) + + if mime.startswith("text/") and not "charset" in f.mime: + mime += "; charset=utf-8" + + ext = os.path.splitext(f.filename)[1] + + if not ext: + gmime = mime.split(";")[0] + + if not gmime in app.config["FHOST_EXT_OVERRIDE"]: + ext = guess_extension(gmime) + else: + ext = app.config["FHOST_EXT_OVERRIDE"][gmime] + else: + ext = ext[:8] + + if not ext: + ext = ".bin" + + with open(getpath(digest), "wb") as of: + of.write(data) + + sf = File(digest, ext, mime, addr) + db.session.add(sf) + db.session.commit() + + return geturl(sf.getname()) + +def store_url(url, addr): + fhost_url = url_for(".fhost", _external=True).rstrip("/") + fhost_url_https = url_for(".fhost", _external=True, _scheme="https").rstrip("/") + + if url.startswith(fhost_url) or url.startswith(fhost_url_https): + return segfault(508) + + r = requests.get(url, stream=True, verify=False) + + try: + r.raise_for_status() + except (requests.exceptions.HTTPError, e): + return str(e) + "\n" + + if "content-length" in r.headers: + l = int(r.headers["content-length"]) + + if l < app.config["MAX_CONTENT_LENGTH"]: + def urlfile(**kwargs): + return type('',(),kwargs)() + + f = urlfile(stream=r.raw, content_type=r.headers["content-type"], filename="") + + return store_file(f, addr) + else: + hl = naturalsize(l, binary = True) + hml = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) + + return "Remote file too large ({0} > {1}).\n".format(hl, hml), 413 + else: + return "Could not determine remote file size (no Content-Length in response header; shoot admin).\n", 411 + +@app.route("/") +def get(path): + p = os.path.splitext(path) + id = su.debase(p[0]) + + if p[1]: + f = File.query.get(id) + + if f and f.ext == p[1]: + if f.removed: + return legal() + + fpath = getpath(f.sha256) + + if not os.path.exists(fpath): + abort(404) + + fsize = os.path.getsize(fpath) + + if app.config["FHOST_USE_X_ACCEL_REDIRECT"]: + response = make_response() + response.headers["Content-Type"] = f.mime + response.headers["Content-Length"] = fsize + response.headers["X-Accel-Redirect"] = "/" + fpath + return response + else: + return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) + else: + u = URL.query.get(id) + + if u: + return redirect(u.url) + + abort(404) + +@app.route("/", methods=["GET", "POST"]) +def fhost(): + if request.method == "POST": + sf = None + + if "file" in request.files: + return store_file(request.files["file"], request.remote_addr) + elif "url" in request.form: + return store_url(request.form["url"], request.remote_addr) + elif "shorten" in request.form: + return shorten(request.form["shorten"]) + + abort(400) + else: + fmts = list(app.config["FHOST_EXT_OVERRIDE"]) + fmts.sort() + maxsize = naturalsize(app.config["MAX_CONTENT_LENGTH"], binary=True) + maxsizenum, maxsizeunit = maxsize.split(" ") + maxsizenum = float(maxsizenum) + maxsizehalf = maxsizenum / 2 + + if maxsizenum.is_integer(): + maxsizenum = int(maxsizenum) + if maxsizehalf.is_integer(): + maxsizehalf = int(maxsizehalf) + + return """
+THE NULL POINTER
+================
+
+HTTP POST files here:
+    curl -F'file=@yourfile.png' {0}
+You can also POST remote URLs:
+    curl -F'url=http://example.com/image.jpg' {0}
+Or you can shorten URLs:
+    curl -F'shorten=http://example.com/some/long/url' {0}
+
+File URLs are valid for at least 30 days and up to a year (see below).
+Shortened URLs do not expire.
+
+Maximum file size: {1}
+Not allowed: {5}
+
+
+FILE RETENTION PERIOD
+---------------------
+
+retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
+
+   days
+    365 |  \\
+        |   \\
+        |    \\
+        |     \\
+        |      \\
+        |       \\
+        |        ..
+        |          \\
+  197.5 | ----------..-------------------------------------------
+        |             ..
+        |               \\
+        |                ..
+        |                  ...
+        |                     ..
+        |                       ...
+        |                          ....
+        |                              ......
+     30 |                                    ....................
+          0{2}{3}
+           {4}
+
+
+ABUSE
+-----
+
+If you would like to request permanent deletion, please contact lachs0r via
+IRC on Freenode, or send an email to lachs0r@(this domain).
+
+Please allow up to 24 hours for a response.
+
+""".format(url_for(".fhost", _external=True).rstrip("/"), + maxsize, str(maxsizehalf).rjust(27), str(maxsizenum).rjust(27), + maxsizeunit.rjust(54), + ", ".join(app.config["FHOST_MIME_BLACKLIST"])) + +@app.route("/robots.txt") +def robots(): + return """User-agent: * +Disallow: / +""" + +def legal(): + return "451 Unavailable For Legal Reasons\n", 451 + +@app.errorhandler(400) +@app.errorhandler(404) +@app.errorhandler(414) +@app.errorhandler(415) +def segfault(e): + return "Segmentation fault\n", e.code + +@app.errorhandler(404) +def notfound(e): + return u"""
Process {0} stopped
+* thread #1: tid = {0}, {1:#018x}, name = '{2}'
+    frame #0:
+Process {0} stopped
+* thread #8: tid = {0}, {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139, name = 'fhost/responder', stop reason = invalid address (fault address: 0x30)
+    frame #0: {3:#018x} fhost`get(path='{4}') + 27 at fhost.c:139
+   136   get(SrvContext *ctx, const char *path)
+   137   {{
+   138       StoredObj *obj = ctx->store->query(shurl_debase(path));
+-> 139       switch (obj->type) {{
+   140           case ObjTypeFile:
+   141               ctx->serve_file_id(obj->id);
+   142               break;
+(lldb) q
+""".format(os.getpid(), id(app), "fhost", id(get), escape(request.path)), e.code + +@manager.command +def debug(): + app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False + app.run(debug=True, port=4562,host="0.0.0.0") + +@manager.command +def permadelete(name): + id = su.debase(name) + f = File.query.get(id) + + if f: + if os.path.exists(getpath(f.sha256)): + os.remove(getpath(f.sha256)) + f.removed = True + db.session.commit() + +@manager.command +def query(name): + id = su.debase(name) + f = File.query.get(id) + + if f: + print("url: {}".format(f.getname())) + vals = vars(f) + + for v in vals: + if not v.startswith("_sa"): + print("{}: {}".format(v, vals[v])) + +@manager.command +def queryhash(h): + f = File.query.filter_by(sha256=h).first() + if f: + query(su.enbase(f.id, 1)) + +@manager.command +def queryaddr(a): + res = File.query.filter_by(addr=a) + + for f in res: + query(su.enbase(f.id, 1)) + +if __name__ == "__main__": + manager.run() -- cgit v1.2.3