diff options
38 files changed, 2232 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba95721 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +__pycache__/ +.py[cod] +up/ +upload/ +*.pyc +.*.swp +*.conf +*.ini +*.log +debian/.debhelper/ +debian/*debhelper* +debian/files +debian/*.substvars +debian/fuss/ +.cache +.dbus +*.pid diff --git a/README.md b/README.md new file mode 100644 index 0000000..f09ebe8 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# README for FUSS +FUSS is a File Upload and Storage Service. +Think [0x0](https://github.com/mia-0/0x0) but without the url shortening. This is intended for private use, such as behind password authentication on the web server level. + +## Overview +FUSS runs a web front-end with a little landing page, and some summaries, and you can upload files to the web app. You can download files from it. This project is a demo of the original author's web app skills, and is designed to be relatively bare so you can modify or theme or improve it as you see fit. + +## License +GPL 3.0 + +## How to use +Fuss is a wsgi application. You need some basic python3 packages, and then configure the two config files, which are self-explanatory. + +* fuss.conf +* fuss.wsgi.ini + +After configuring the files, you can run: + + /usr/sbin/fuss.bin + +This short script runs uwsgi within xvfb-run, which is necessary if you want to use mimetype icons (loaded through Gtk). If you omit icons, you can omit the xvfb-run part. + +You can run this with the included [init script](extras/fuss.init) if you prefer to have a system service. + +### Running on certain distros +After installing the package, the admin might need to run a helper script that will install the necessary pip packages. Known situations include: +* CentOS 7: python-flask, python-magic + +Run script: + + sudo /usr/libexec/fuss/pip-helper.sh + +Add the parameter "PyGObject" if you compiled the rpm with icon support (which pulls in a lot of dependencies). + +On Fedora, selinux prevents running /var/www/fuss/fuss.bin by systemd. + +## Dependencies +* python3-flask +* python3-magic +* For icon support (optional), which just add mimetype icons to the html view of files. + * Xvfb-run + * PyGObject + * python3-gi + +## Alternatives +[0x0](https://github.com/mia-0/0x0) or my fork of it, [hex-zero](https://gitlab.com/bgstack15/hex-zero) + +## References +### Overall +Heavily inspired by [0x0](https://github.com/mia-0/0x0) or my fork of it, [hex-zero](https://gitlab.com/bgstack15/hex-zero) +### Specific code snippets +* https://code-boxx.com/simple-drag-and-drop-file-upload/ +* entire flask series at https://pythonise.com/series/learning-flask/flask-uploading-files +* hex_zero.py from https://gitlab.com/bgstack15/hex-zero +* https://stackoverflow.com/a/65433574/3569534 +* https://werkzeug.palletsprojects.com/en/1.0.x/middleware/proxy_fix/ +* https://uwsgi-docs.readthedocs.io/en/latest/Logging.html +* Reset stream when reading file from request: https://stackoverflow.com/a/26294982/3569534 +* https://salsa.debian.org/sssd-team/sssd/-/blob/experimental/debian/sssd-common.postinst +* /home/var/lib/dpkg/info/postfix.prerm +* bgscripts.spec +* https://docs.fedoraproject.org/en-US/packaging-guidelines/UsersAndGroups/ +* https://www.digitalocean.com/community/tutorials/css-collapsible +* viewport in html: https://stackoverflow.com/a/30709473/3569534 +* https://www.thesitewizard.com/css/mobile-friendly-responsive-design.shtml +* https://www.dotnettricks.com/learn/stylesheet/css-to-force-long-text-and-urls-to-wrap-on-all-browser +* From https://stackoverflow.com/a/14226807/3569534 diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..92940d2 --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,7 @@ +fuss for Devuan + +To make fuss work with apache2 in a Debian-like apache2 environment, you will need to run these commands. + + for word in proxy rewrite ssl headers ; do sudo a2enmod ${word} ; done + + -- B. Stack <bgstack15@gmail.com> Tue, 05 Jan 2021 11:04:29 -0500 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..3e9adb2 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +fuss (0.0.1-1) obs; urgency=low + + * Initial release. Closes: packages-want#000 + + -- B. Stack <bgstack15@gmail.com> Tue, 05 Jan 2021 11:04:29 -0500 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..9b08d51 --- /dev/null +++ b/debian/control @@ -0,0 +1,23 @@ +Source: fuss +Section: web +Priority: optional +Maintainer: B. Stack <bgstack15@gmail.com> +Build-Depends: debhelper-compat (= 12) +Standards-Version: 4.5.0 +Homepage: https://gitlab.com/bgstack15/fuss/ + +Package: fuss +Architecture: all +Multi-Arch: foreign +Pre-Depends: adduser +Depends: ${misc:Depends}, ${shlibs:Depends}, + lsb-base, + python3-flask, + python3-uwsgidecorators, + python3:any, + uwsgi-core, + uwsgi-plugin-python3 +Recommends: apache2 +Description: File Upload and Storage Service + A demo web app using flask to present a simple + web page for uploading and downloading files. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..ec90d60 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,39 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: fuss +Upstream-Contact: <preferred name and address to reach the upstream project> +Source: <url://example.com> +# +# Please double check copyright with the licensecheck(1) command. + +Files: README.md + extra/Makefile + extra/fuss.conf.apache + fuss.bin + fuss.conf.example + fuss.py + fuss.wsgi.ini.example + static/files.css + static/fuss-upload + static/index.css + static/index.js + static/robots.txt + static/upload.css + templates/index.html.j2 + templates/upload.html.j2 +Copyright: __NO_COPYRIGHT_NOR_LICENSE__ +License: __NO_COPYRIGHT_NOR_LICENSE__ + +Files: extra/fuss.init +Copyright: __NO_COPYRIGHT__ in: extra/fuss.init +License: __UNKNOWN__ + Do NOT "set -e" + . + PATH should only include /usr/* if it runs after the mountnfs.sh script + +Files: templates/upload.js.j2 +Copyright: AJAX UPLOAD +License: __NO_LICENSE__ + +#---------------------------------------------------------------------------- +# Files marked as NO_LICENSE_TEXT_FOUND may be covered by the following +# license/copyright files. diff --git a/debian/fuss.lintian-overrides b/debian/fuss.lintian-overrides new file mode 100644 index 0000000..78801ac --- /dev/null +++ b/debian/fuss.lintian-overrides @@ -0,0 +1,6 @@ +copyright-has-url-from-dh_make-boilerplate +copyright-without-copyright-notice +dir-or-file-in-var-www +script-in-etc-init.d-not-registered-via-update-rc.d +script-not-executable +uses-dpkg-database-directly prerm diff --git a/debian/fuss_0.0.1-1.dsc b/debian/fuss_0.0.1-1.dsc new file mode 100644 index 0000000..03f8625 --- /dev/null +++ b/debian/fuss_0.0.1-1.dsc @@ -0,0 +1,12 @@ +Format: 3.0 (quilt) +Source: fuss +Binary: fuss +Architecture: all +Version: 0.0.1-1 +Maintainer: B. Stack <bgstack15@gmail.com> +Homepage: https://gitlab.com/bgstack15/fuss/ +Standards-Version: 4.5.0 +Build-Depends: debhelper-compat (= 12) +Files: + 00000000000000000000000000000000 1 fuss.orig.tar.gz + 00000000000000000000000000000000 1 fuss.debian.tar.xz diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000..4a97dfa --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +# You must remove unused comment lines for the released package. diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..9c80e67 --- /dev/null +++ b/debian/postinst @@ -0,0 +1,36 @@ +#!/bin/sh +set -e + +#DEBHELPER# + +OUT=/dev/null +USERNAME=fuss +HOME=/var/www/fuss +SERVICE=fuss + +case "$1" in + configure) + update-rc.d ${SERVICE} defaults + if ! getent passwd ${USERNAME} 1>${OUT} 2>&1 ; + then + echo "Creating ${USERNAME} system user & group..." + adduser --quiet --system --home $HOME \ + --disabled-password --group \ + --gecos "FUSS system user" \ + ${USERNAME} > ${OUT} 2>&1 + fi + chown ${USERNAME}:${USERNAME} \ + ${HOME} ${HOME}/* \ + /var/log/${SERVICE} /var/log/${SERVICE}/*log 1>/${OUT} 2>&1 || : + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/debian/postrm b/debian/postrm new file mode 100644 index 0000000..29efbb0 --- /dev/null +++ b/debian/postrm @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +#DEBHELPER# +OUT=/dev/null +HOME=/var/www/fuss +SERVICE=fuss + +case "$1" in + purge) + update-rc.d -f ${SERVICE} remove + rm -f ${HOME}/upload/.*.meta 1>${OUT} 2>&1 || : + ;; + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/debian/prerm b/debian/prerm new file mode 100644 index 0000000..a0e1dcb --- /dev/null +++ b/debian/prerm @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +OUT=/dev/null +HOME=/var/www/fuss +SERVICE=fuss + +case "$1" in + remove) + test -x /usr/sbin/invoke-rc.d && \ + INIT="invoke-rc.d ${SERVICE}" || \ + INIT="/etc/init.d/${SERVICE}" + ${INIT} stop + ;; + + purge) + rm -f ${HOME}/fuss.pid 1>${OUT} 2>&1 || : + ;; + + upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..4da7afe --- /dev/null +++ b/debian/rules @@ -0,0 +1,14 @@ +#!/usr/bin/make -f +# You must remove unused comment lines for the released package. +#export DH_VERBOSE = 1 +#export DEB_BUILD_MAINT_OPTIONS = hardening=+all +#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic +#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed + +%: + dh $@ --sourcedirectory=extra + +override_dh_auto_install: + dh_auto_install -- prefix=/usr \ + DEFAULTDIR='$$(DESTDIR)/etc/default' \ + APACHEDIR='$$(SYSCONFDIR)/apache2/sites-available' diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 0000000..41c1457 --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,3 @@ +syntax-error-in-dep5-copyright +file-without-copyright-information +missing-license-paragraph-in-dep5-copyright diff --git a/debian/source/local-options b/debian/source/local-options new file mode 100644 index 0000000..00131ee --- /dev/null +++ b/debian/source/local-options @@ -0,0 +1,2 @@ +#abort-on-upstream-changes +#unapply-patches diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..76575dc --- /dev/null +++ b/debian/watch @@ -0,0 +1,2 @@ +# You must remove unused comment lines for the released package. +version=3 diff --git a/extra/Makefile b/extra/Makefile new file mode 100644 index 0000000..d18b75a --- /dev/null +++ b/extra/Makefile @@ -0,0 +1,128 @@ +# File: Makefile for fuss +# Location: fuss source package +# Author: bgstack15 +# Startdate: 2020-12-30 +# Title: Makefile for fuss source package +# Purpose: To use traditional Unix make utility +# History: +# Usage: +# Reference: +# hex-zero Makefile +# Improve: +# add man page? +# Document: +# Dependencies: +# build-devuan: + +APPNAME = fuss +APPVERSION = 0.0.1 +SRCDIR = $(CURDIR)/..# because Makefile is in extra/ inside this repo +prefix = /usr +SYSCONFDIR = $(DESTDIR)/etc +DEFAULTDIR = $(DESTDIR)/etc/sysconfig# for debian use '$(DESTDIR)/etc/default' +LIBEXECDIR = $(DESTDIR)$(prefix)/libexec +SHAREDIR = $(DESTDIR)$(prefix)/share +DOCDIR = $(SHAREDIR)/doc/$(APPNAME) +APPDIR = $(SHAREDIR)/$(APPNAME) +APPVARDIR = $(DESTDIR)/var/www/$(APPNAME) +MANDIR = $(SHAREDIR)/man +SYSVDIR = $(SYSCONFDIR)/init.d +SYSDDIR = $(DESTDIR)$(prefix)/lib/systemd/system +LOGDIR = $(DESTDIR)/var/log/fuss +APACHEDIR = $(SYSCONFDIR)/httpd/conf.d# for debian use '$(SYSCONFDIR)/apache2/sites-available' +CRONDIR = $(SYSCONFDIR)/cron.d +SBINDIR = $(DESTDIR)$(prefix)/sbin + +# variables for deplist +DEPTYPE = dep +SEPARATOR = , + +awkbin :=$(shell which awk) +chmodbin :=$(shell which chmod) +cpbin :=$(shell which cp) +echobin :=$(shell which echo) +falsebin :=$(shell which false) +findbin :=$(shell which find) +grepbin :=$(shell which grep) +gzipbin :=$(shell which gzip) +installbin :=$(shell which install) +rmbin :=$(shell which rm) +rmdirbin :=$(shell which rmdir) +sedbin :=$(shell which sed) +sortbin :=$(shell which sort) +truebin :=$(shell which true) +uniqbin :=$(shell which uniq) +xargsbin :=$(shell which xargs) + +with_apache ?= YES +with_init ?= YES +with_systemd ?= NO +with_pip_helper ?= NO + +all: + -@echo "Nothing to build." && ${truebin} + +install: install_files + +.PHONY: clean install install_files uninstall list deplist deplist_opts + +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | ${awkbin} -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | ${sortbin} | ${grepbin} -E -v -e '^[^[:alnum:]]' -e '^$@$$' + +deplist: + @# deplist 2020-04-18 input must be comma separated + @# DEPTYPE( dep , rec , sug ) for depends, recommends, or suggests + @if test -z "${DISTRO}" ; then ${echobin} "Please run \`make deplist\` with DISTRO= one of: `make deplist_opts 2>&1 1>/dev/null | ${xargsbin}`. Aborted." 1>&2 ; exit 1 ; fi + @if ! ${echobin} "${DEPTYPE}" | grep -qE "^(dep|rec|sug)$$" ; then ${echobin} "Please run \`make deplist\` with DEPTYPE= one of: dep, rec, sug. Undefined will use \`dep\`. Aborted." 1>&2 ; exit 1; fi + @${grepbin} -h --exclude-dir='doc' -riIE "\<${DEPTYPE}-" ${SRCDIR} | ${awkbin} -v "domain=${DISTRO}" -v "deptype=${DEPTYPE}" 'tolower($$2) ~ deptype"-"domain {$$1="";$$2="";print}' | tr ',' '\n' | ${sortbin} | ${uniqbin} | ${sedbin} -r -e 's/^\s*//' -e "s/\s*\$$/${SEPARATOR}/" | ${xargsbin} + +deplist_opts: + @# deplist_opts 2020-04-18 find all available dependency domains + @${grepbin} -h -o -riIE '\<(dep|rec|sug)-[^\ :]+:' ${SRCDIR} | ${sedbin} -r -e 's/(dep|rec|sug)-//;' -e 's/:$$//;' | ${sortbin} | ${uniqbin} 1>&2 + +install_files: + ${installbin} -m0755 -d ${LOGDIR} ${APPVARDIR}/upload ${APPDIR}/static ${APPDIR}/templates \ + ${DOCDIR} ${SBINDIR} ${SYSCONFDIR} ${LIBEXECDIR}/${APPNAME} + ${installbin} -m0755 -t ${SBINDIR} ${SRCDIR}/${APPNAME}.bin + ${installbin} -m0644 -t ${LIBEXECDIR}/${APPNAME} ${SRCDIR}/${APPNAME}.py + ${installbin} -m0644 ${SRCDIR}/${APPNAME}.conf.example ${SYSCONFDIR}/${APPNAME}.conf + ${installbin} -m0644 ${SRCDIR}/${APPNAME}.wsgi.ini.example ${SYSCONFDIR}/${APPNAME}.wsgi.ini + ${installbin} -m0644 -t ${DOCDIR} ${SRCDIR}/*.md + ${installbin} -m0644 -t ${APPDIR}/static ${SRCDIR}/static/* + ${installbin} -m0644 -t ${APPDIR}/templates ${SRCDIR}/templates/* +ifeq ($(with_apache),YES) + ${installbin} -m0755 -d ${APACHEDIR} + ${installbin} -m0644 ${SRCDIR}/extra/${APPNAME}.conf.apache ${APACHEDIR}/${APPNAME}.conf +endif +ifeq ($(with_init),YES) + ${installbin} -m0755 -d ${SYSVDIR} + ${installbin} -m0755 ${SRCDIR}/extra/${APPNAME}.init ${SYSVDIR}/${APPNAME} +endif +ifeq ($(with_systemd),YES) + ${installbin} -m0755 -d ${SYSDDIR} + ${installbin} -m0644 ${SRCDIR}/extra/fuss.service -t ${SYSDDIR} +endif +ifeq ($(with_pip_helper),YES) + ${installbin} -m0755 -t ${LIBEXECDIR}/${APPNAME}/ ${SRCDIR}/extra/pip-helper.sh +endif + +uninstall: + @${echobin} SRCDIR=${SRCDIR} + ${rmbin} -f ${APPDIR}/${APPNAME}.* \ + ${DOCDIR}/* \ + ${APPDIR}/static/* \ + ${APPDIR}/templates/* \ + ${LIBEXECDIR}/${APPNAME}/* \ + ${SBINDIR}/${APPNAME}.bin \ + 1>/dev/null 2>&1 || : + ${rmbin} -f ${APACHEDIR}/${APPNAME}.conf || : + ${rmbin} -f ${SYSVDIR}/${APPNAME} || : + + # remove all installed directories that are now blank. + ${rmdirbin} ${APPVARDIR}/upload ${APPDIR}/static \ + ${SYSVDIR} ${APACHEDIR} ${DOCDIR} \ + ${LIBEXECDIR}/${APPNAME} ${LOGDIR} 2>/dev/null || : + ${rmdirbin} ${APPDIR} 2>/dev/null || : + +clean: + -@${echobin} "target $@ not implemented yet! Gotta say unh." && ${falsebin} diff --git a/extra/fuss-el7.patch b/extra/fuss-el7.patch new file mode 100644 index 0000000..3a9ceed --- /dev/null +++ b/extra/fuss-el7.patch @@ -0,0 +1,28 @@ +Summary: Make changes necessary for this package to work on CentOS 7 +Message: Correct name of python36 wsgi plugin, and disable icons +Date: 2021-01-05 +diff master/fuss.wsgi.ini.example el7/fuss.wsgi.ini.example +index 42ebfe2..75b59bb 100644 +--- a/fuss.wsgi.ini.example ++++ b/fuss.wsgi.ini.example +@@ -2,7 +2,7 @@ + # References: + # https://uwsgi-docs.readthedocs.io/en/latest/Logging.html + [uwsgi] +-plugins = python3,logfile ++plugins = python36,logfile + http-socket = 0.0.0.0:5003 + wsgi-file = /usr/libexec/fuss/fuss.py + callable = app +--- a/fuss.conf.example 2021-01-05 22:20:05.102222226 -0500 ++++ b/fuss.conf.example 2021-01-05 22:24:51.642095995 -0500 +@@ -52,7 +52,7 @@ + ] + + # Include mimetype icon support on html-long. This requires META = True +-ICONS = True ++ICONS = False + ICON_THEME = "Numix" + + # for in-app adjustments of logging, see project [hex-zero](https://gitlab.com/bgstack15/hex-zero/-/tree/stack) which has limited ability to affect logging. + diff --git a/extra/fuss.conf.apache b/extra/fuss.conf.apache new file mode 100644 index 0000000..661b708 --- /dev/null +++ b/extra/fuss.conf.apache @@ -0,0 +1,60 @@ +# Apache example config for fuss application +<VirtualHost *:80> + + ServerName d2-03a.ipa.example.com + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + #LogLevel info ssl:warn + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + # OPTION 1: send to https + # force https for this path + RewriteEngine On + RewriteCond %{HTTPS} !=on + RewriteCond %{HTTP_HOST} !^(localhost|127.0.0.1) + RewriteRule ^/fuss(.*) https://%{SERVER_NAME}/fuss$1 [R,L] + + # OPTION 2: Just use unencrypted + #ProxyPass /fuss http://localhost:5003/ + #ProxyPassReverse /fuss http://localhost:5003/ + #<Location /fuss> + # RequestHeader append X-Forwarded-Prefix "/fuss" + # RequestHeader set X-Forwarded-Proto "http" + #</Location> + +</VirtualHost> + +# To use OPTION 2 above, just disable this whole 443 virtualhost. +<VirtualHost *:443> + ServerName d2-03a.ipa.example.com + + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + #LogLevel info ssl:warn + + ErrorLog ${APACHE_LOG_DIR}/ssl-error.log + CustomLog ${APACHE_LOG_DIR}/ssl-access.log combined + + SSLEngine on + SSLProtocol all -SSLv2 -SSLv3 + SSLHonorCipherOrder on + SSLCipherSuite "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS" + + SSLCertificateFile /etc/ssl/private/https-d2-03a.ipa.example.com.pem + SSLCertificateKeyFile /etc/ssl/private/https-d2-03a.ipa.example.com-nopw.key + + ProxyPass /fuss http://localhost:5003/ + ProxyPassReverse /fuss http://localhost:5003/ + <Location /fuss> + # a2enmod headers. These are extra ones that are not provided by Apache natively. + RequestHeader append X-Forwarded-Prefix "/fuss" + RequestHeader set X-Forwarded-Proto "https" + </Location> + +</VirtualHost> +# vim:set syntax=apache ts=3 sw=3 sts=3 sr noet: diff --git a/extra/fuss.init b/extra/fuss.init new file mode 100755 index 0000000..7a7b5d7 --- /dev/null +++ b/extra/fuss.init @@ -0,0 +1,172 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: fuss +# Required-Start: $local_fs $network $remote_fs $syslog +# Required-Stop: $local_fs $network $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: File Upoad and Storage Service +# Description: Python3 Flask application that presents a basic file hosting service +### END INIT INFO + +# Author: B. Stack <bgstack15@gmail.com> + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +DAEMON_ARGS="" +DAEMON=/usr/sbin/fuss.bin +DESC="fuss" +NAME=fuss +PATH=/sbin:/usr/sbin:/bin:/usr/bin +PIDFILE2=/var/www/fuss/fuss.pid +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +USER=fuss +WSGIBIN=uwsgi_python39 + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + #start-stop-daemon --start --quiet --chuid $USER --group $USER --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \ + # || return 1 + #start-stop-daemon --start --quiet --chuid $USER --group $USER --background --pidfile $PIDFILE --exec $DAEMON -- \ + # $DAEMON_ARGS \ + # || return 2 + #su $USER -c "$DAEMON" & + # The above code will not work for interpreted scripts, use the next + # six lines below instead (Ref: #643337, start-stop-daemon(8) ) + start-stop-daemon --start --make-pidfile --pidfile $PIDFILE \ + --chuid $USER --group $USER \ + --startas $DAEMON \ + --name $WSGIBIN --test > /dev/null \ + || return 1 + start-stop-daemon --start --background --make-pidfile --pidfile $PIDFILE --startas $DAEMON \ + --chuid $USER --group $USER \ + --name $WSGIBIN -- $DAEMON_ARGS \ + || return 2 + + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + #start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --user $USER --pidfile $PIDFILE + /usr/bin/$WSGIBIN --stop $PIDFILE2 2>/dev/null + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --user $USER --pidfile $PIDFILE + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $WSGIBIN + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/extra/fuss.service b/extra/fuss.service new file mode 100644 index 0000000..709ded6 --- /dev/null +++ b/extra/fuss.service @@ -0,0 +1,20 @@ +[Unit] +Description=File Upload and Storage Service +Wants=network-online.target +After=network.target syslog.target + +[Service] +#Environment= +WorkingDirectory=/var/www/fuss +User=fuss +Group=fuss +Type=simple +ExecStart=/usr/sbin/fuss.bin +TimeoutStartSec=120 +ExecStop=/usr/sbin/uwsgi --stop /var/www/fuss/fuss.pid +RestartSec=15 +Restart=always +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target diff --git a/extra/fuss.spec b/extra/fuss.spec new file mode 100644 index 0000000..3519216 --- /dev/null +++ b/extra/fuss.spec @@ -0,0 +1,167 @@ +# File: fuss.spec +# Location: fuss package +# Author: bgstack15 +# SPDX-License-Identifier: CC-BY-SA-4.0 +# Startdate: 2021-01-05 +# Title: Rpm spec for fuss package +# Purpose: Provide build instructions for CentOS rpm for package +# History: +# Usage: +# Reference: +# Improve: +# Documentation: +# Dependencies: + +# Tunables +# If you set with_icons to 1, then you will pull in a large amount of dependencies! +%global with_icons 0 +# If you set this to 1, use the systemd-rpm-macros functionality described at https://docs.fedoraproject.org/en-US/packaging-guidelines/UsersAndGroups/ +%global with_systemd_usercreate 0 +%global pythonver python36 + +# Fedora defaults +%if 0%{?fedora} +%global with_systemd_usercreate 1 +%global with_icons 1 +%global pythonver python3 +%endif + +%global _appvardir %{?_localstatedir}%{!?_localstatedir:/var}/www/fuss +%global _appdir %{?_datarootdir}%{!?_datarootdir:%{_prefix}/share}/fuss +%global _user fuss + +%define devtty "/dev/null" +%define debug_package %{nil} +%global _python_bytecompile_errors_terminate_build 0 + +Summary: file upload and storage service web app +Name: fuss +Version: 0.0.1 +Release: 1 +License: GPL 3.0 +Source0: %{name}_%{version}.orig.tar.gz +%if 0%{?with_systemd_usercreate} +Source1: extra/%{name}.sysusers +%endif +%if ! 0%{?fedora} +Patch1: extra/%{name}-el7.patch +%endif +URL: https://bgstack15.wordpress.com/ +#Distribution: +#Vendor: +Packager: B. Stack <bgstack15@gmail.com> +Requires: %{pythonver}-flask +Requires: %{pythonver}-uwsgidecorators +Requires: uwsgi-plugin-%{pythonver} +Requires: uwsgi-logger-file +%if 0%{?fedora} +Requires: %{pythonver}-magic +Requires: %{pythonver}-flask-script +%endif +# Mandatory pip3 requirements: flask-script, python-magic +%if 0%{?with_icons} +Requires: /usr/bin/xvfb-run +%if 0%{?fedora} +Requires: %{pythonver}-gobject +%else +# The following are needed for CentOS7 pip3 install --user PyGObject +Requires: cairo-gobject-devel +Requires: gcc +Requires: gobject-introspection-devel +Requires: gtk3-devel +Requires: python36-gobject +Requires: python3-devel +%endif +%endif +#BuildRequires: txt2man +%if 0%{?with_systemd_usercreate} +BuildRequires: systemd-rpm-macros +%endif +%if 0%{?fedora} || 0%{?rhel} >= 8 +Suggests: httpd +%endif +Buildarch: noarch + +%description +File Upload and Storage Service is a demo flask application that can +be used for whatever purposes you want. + +%prep +%setup -q -c %{name} +test -d "%{name}" && cd "%{name}" +%if ! 0%{?fedora} +%patch1 -p1 +%endif + +%build +export srcdir="extra" +test -d "%{name}" && cd "%{name}" +%make_build -C "${srcdir}" + +%install +export srcdir="extra" +test -d "%{name}" && cd "%{name}" +%make_install -C "${srcdir}" \ + with_systemd=YES \ + with_init=NO \ + with_pip_helper=YES +%if 0%{?with_systemd_usercreate} +install -p -D -m 0644 %{SOURCE1} %{buildroot}%{_sysusersdir}/%{name}.conf +%endif +exit 0 + +%clean +rm -rf %{buildroot} + +%pre +# Reference: squid.spec +%if 0%{?with_systemd_usercreate} +%sysusers_create_compat %{SOURCE1} +%else +if ! getent group %{_user} 1>/dev/null 2>&1 ; +then + /usr/sbin/groupadd --system --gid 723 %{_user} +fi +if ! getent passwd %{_user} 1>/dev/null 2>&1 ; +then + /usr/sbin/useradd --system --gid 723 \ + --uid 723 --comment "FUSS system user" \ + --home-dir %{_appvardir} --shell /sbin/nologin \ + %{_user} +fi +%endif +exit 0 + +%preun +%systemd_postun_with_restart %{name}.service + +%post +%systemd_post %{name}.service + +%postun +%systemd_postun_with_restart %{name}.service + +%files +%if 0%{?with_systemd_usercreate} +%{_sysusersdir}/%{name}.conf +%endif +%attr(0644, %{_user}, %{_user}) %config(noreplace) %{_sysconfdir}/%{name}.conf +%attr(0644, %{_user}, %{_user}) %config(noreplace) %{_sysconfdir}/%{name}.wsgi.ini +%attr(0644, %{_user}, %{_user}) %{_libexecdir}/%{name}/%{name}.py +%attr(0755, %{_user}, %{_user}) %{_sbindir}/%{name}.bin +%attr(0644, %{_user}, %{_user}) %{_appdir}/static/* +%attr(0644, %{_user}, %{_user}) %{_appdir}/templates/* +%attr(0755, %{_user}, %{_user}) %dir %{_appdir}/static +%attr(0755, %{_user}, %{_user}) %dir %{_appdir}/templates +%attr(0755, %{_user}, %{_user}) %dir %{_appvardir}/upload +%attr(0755, %{_user}, %{_user}) %dir %{_appdir} +%attr(0755, %{_user}, %{_user}) %dir %{_appvardir} +%attr(0755, -, -) %{_libexecdir}/%{name}/pip-helper.sh +%attr(0644, root, root) %{?_unitdir}%{!?_unitdir:/usr/lib/systemd/system}/%{name}.service +%attr(0644, root, root) %{_sysconfdir}/httpd/conf.d/%{name}.conf +%attr(0755, %{_user}, %{_user}) %dir %{?_localstatedir}%{!?_localstatedir:/var}/log/%{name} +%{_defaultdocdir}/%{name} + +%changelog +* Tue Jan 05 2021 B. Stack <bgstack15@gmail.com> - 0.0.1-1 +- Initial release diff --git a/extra/fuss.sysusers b/extra/fuss.sysusers new file mode 100644 index 0000000..e9f60df --- /dev/null +++ b/extra/fuss.sysusers @@ -0,0 +1,3 @@ +# Part of fuss package +#Type Name ID GECOS Home directory Shell +u fuss - "FUSS system user" /var/www/fuss /sbin/nologin diff --git a/extra/pip-helper.sh b/extra/pip-helper.sh new file mode 100644 index 0000000..39213aa --- /dev/null +++ b/extra/pip-helper.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# This file is part of the fuss package. +# It is designed to be run as root, and it will run the requisite pip commands for the discovered environment. + +contents="$( awk -F'=' '/ID=|VERSION_ID/{print $2}' /etc/os-release 2>/dev/null | sed -r -e 's/"//g;' )" +id="$( echo "${contents}" | sed -n -e '1p' )" +version_id="$( echo "${contents}" | sed -n -e '2p' )" + +echo "Any parameters sent to this script ${0} will be added to the list of packages to install." +echo "This is particularly useful for adding the PyGObject for icon support on EL7." +piplist="${1}" + +if echo "${id}" | grep -qiE 'rhel|centos' ; +then + if echo "${version_id}" | grep -qiE '7' ; + then + piplist="${piplist} flask-script python-magic" + fi +else + echo "Unknown os from /etc/os-release. Please investigate what pip3" 1>&2 + echo "packages are required for this OS release and share it with upstream." 1>&2 + echo "Aborted." 1>&2 + exit 1 +fi + +if test -n "${piplist}" ; +then + echo "Will try to serially install with pip these packages: ${piplist}" + for word in ${piplist} ; do + su fuss -s /bin/sh -c "pip3 install --user ${word}" || exit 1 + done +fi diff --git a/fuss.bin b/fuss.bin new file mode 100755 index 0000000..c1f32b7 --- /dev/null +++ b/fuss.bin @@ -0,0 +1,10 @@ +#!/bin/sh +# At a very basic level, it is simply +# xvfb-run uwsgi_python39 --ini /etc/fuss.wsgi.ini +COMMAND="" +grep -qiE 'ICONS\s*=\s*(Yes|True)' "/etc/fuss.conf" 1>/dev/null 2>&1 && COMMAND="xvfb-run" + +# On rhel-like, use "uwsgi" but uwsgi_python39 elsewhere +grep -qiE 'ID=.*(rhel|centos|fedora)' /etc/os-release && COMMAND="${COMMAND} uwsgi" || \ + COMMAND="${COMMAND} uwsgi_python39" +${COMMAND} --ini /etc/fuss.wsgi.ini diff --git a/fuss.conf.example b/fuss.conf.example new file mode 100644 index 0000000..a45dc21 --- /dev/null +++ b/fuss.conf.example @@ -0,0 +1,64 @@ +# File: fuss.conf.example + +# See also: fuss.wsgi.ini + +# Store uploaded files here +UPLOAD_PATH = "/var/www/fuss/upload/" + +# Nginx option that has not yet been tested. +USE_X_ACCEL_REDIRECT = False + +# turn this on to trust the request headers for the incoming path. You can use this instead of APP_SERVER, APP_PREFIX, and PROXY_LAYERS. +DISCOVER_URL = True +# Or manually define what names to use. +#APP_SERVER = "http://www.example.com:5003/" # protocol, host, port +#APP_PREFIX = "" # directory on web host +# And these when behind a reverse proxy +#APP_SERVER = "https://www.example.com" +#APP_PREFIX = "/fuss/" +#PROXY_LAYERS = 1 # must be precise + +# Refuse to store files of these mimetypes. +MIMETYPE_BLACKLIST = [ + "audio/ogg" +] + +# for vanity directories for uploads and downloads. +# if you leave UPLOAD_PREFIX blank, then the upload.html.j2 is never available and only the front page is available to browsers. Uploads are possible, just as a POST to the main application URL. +# These are directories, and the input is sanitized to safely handle if you put slashes here or not. +UPLOAD_PREFIX = "/u/" +DOWNLOAD_PREFIX = "/d/" + +MAX_FILE_SIZE = 1024 * 1024 * 512 # 512MB + +# How many copies of a file to accept +MAX_DUPLICATE_NAMES = 5 + +# not used by wsgi, but useful for when running with python3 +APP_HOST = "0.0.0.0" +APP_PORT = 5003 + +# Run maintenance every X seconds. +LOOP_DELAY = 3 * 60 + +# Store and use extra metadata +META = True +# Allow clients to query metadata by adding ".meta" to filename. This is required for meta_headers as well. +META_VISIBLE = True +# Show the following metadata fields as headers +META_HEADERS = [ + "address", + "uploaded" +] + +# Include mimetype icon support on html-long. This requires META = True +ICONS = True +ICON_THEME = "Numix" + +# for in-app adjustments of logging, see project [hex-zero](https://gitlab.com/bgstack15/hex-zero/-/tree/stack) which has limited ability to affect logging. +# See also https://flask.palletsprojects.com/en/master/logging/ +# But for now, file fuss.wsgi.ini controls the logging. + +# Control what paths are used for source files +TEMPLATE_FOLDER = "/usr/share/fuss/templates" +STATIC_FOLDER = "/usr/share/fuss/static" @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +# File: fuss.py +# Location: fuss source +# Author: bgstack15 +# SPDX-License-Identifier: GPL-3.0 +# Startdate: 2020-12-21 +# Title: File Upload and Storage Service main app +# Purpose: Demonstrate web technologies +# Usage: Use /usr/sbin/fuss.bin or system service +# References: +# Improve: +# Dependencies: +# python3-flask-script, python3-magic, python3-uwsgidecorators +# vim:set ts=3 sw=3 sts=3 et: + +from flask import Flask, make_response, Response, abort, send_from_directory, render_template, request, redirect, url_for +from werkzeug.datastructures import FileStorage +from werkzeug.utils import secure_filename +from werkzeug.middleware.proxy_fix import ProxyFix +from flask_script import Manager, Server # python3-flask-script +import magic # python3-magic, see below +from hashlib import sha256 +from mimetypes import guess_extension +import os, sys, time, json, base64 +from datetime import datetime, date +from uwsgidecorators import * # python3-uwsgidecorators +from logging.config import dictConfig + +app = Flask(__name__) +manager = Manager(app) + +############################################ +# Functions +def now(): + return int(time.mktime(datetime.today().timetuple())) + +def all_same(items): + return all(x == items[0] for x in items) + +def trim_dict(a,max_length=40): + """ Shorten long strings in a dictionary, primarily for displaying partial contents of icondata metadata """ + b={} + for i in a: + if isinstance(a[i], str): + if len(a[i]) > max_length: + b[i] = a[i][0:max_length] + "..." + else: + b[i] = a[i] + return b + +# load config file +# Load it from the current directory, which is not FHS-compliant +#conf_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),"fuss.conf") +conf_file = "/etc/fuss.conf" +app.config.from_pyfile(conf_file, silent=True) +if "WSGI_LOGGING" in app.config: + dictConfig(app.config["WSGI_LOGGING"]) + +# Load gtk only if requested, because who really wants to load that if it is not necessary? +if "ICONS" in app.config and app.config["ICONS"]: + import gi # python3-gi + gi.require_version("Gtk", "3.0") + from gi.repository import Gio, Gtk + +# alias the config values +up = app.config["UPLOAD_PATH"] +ulp = app.config["UPLOAD_PREFIX"] +dlp = app.config["DOWNLOAD_PREFIX"] +aps = "" +try: + aps = app.config["APP_SERVER"] +except: + pass +ap = "" +try: + ap = app.config["APP_PREFIX"] +except: + pass + +ap_s = ap.strip("/") + "/" +s_ap_s = "/" + ap.strip("/") + "/" +ulp_s = ulp.strip("/") + "/" +aps_ap = aps.strip("/") + str("/" + ap.strip("/") + "/").replace("//","/") +ap_ulp = "/" + ap.strip("/") + "/" + ulp.strip("/") + "/" +aps_ap_ulp = aps_ap + ulp_s +aps_ap_dlp = aps_ap + dlp.strip("/") + "/" + +# these values are for internal use only +app.config["ACCEPTED_FORMATS"] = ["text","json","xml","html","html-long"] +mime_dict = { + "json": "application/json", + "text": "text/plain", + "xml": "application/xml", + "html": "text/html", + "html-long": "text/html" +} + +app.config["VISIBLE_CONFIGS"] = [ + "APP_HOST", + "APP_PORT", + "APP_PREFIX", + "DOWNLOAD_PREFIX", + "UPLOAD_PREFIX", + "UPLOAD_PATH", + "MIMETYPE_BLACKLIST", + "MAX_FILE_SIZE", + "MAX_DUPLICATE_NAMES", + "USE_X_ACCEL_REDIRECT", + "APP_SERVER", + "LOOP_DELAY", + "META", + "META_VISIBLE", + "META_HEADERS", + "ICONS", + "ICON_THEME" +] + +try: + mimedetect = magic.Magic(mime=True, mime_encoding=False) + USE_DARWIN_MAGIC = False +except: + # then perhaps we are using the Fedora-packaged magic which for some reason is from a different source. + # Debian uses https://github.com/ahupp/python-magic/ + # Fedora uses http://www.darwinsys.com/file/ + import magic + USE_DARWIN_MAGIC = True +# print("""Error: You have installed the wrong version of the 'magic' module. +#Please install python-magic.""") +# sys.exit(1) + +def detect_from_file(filename): + if not USE_DARWIN_MAGIC: + # ahupp way + #return mimedetect.from_buffer(filename) + return mimedetect.from_file(filename) + else: + return magic.detect_from_filename(filename).mime_type + return None + +def detect_from_buffer(buffer): + if not USE_DARWIN_MAGIC: + return mimedetect.from_buffer(buffer) + else: + return magic.detect_from_content(buffer) + return "sample" + +# because the x_for level must be precise. +# Apache needs RequestHeader statements like so, assuming this is in a block with SSLEngine on. +# <Location /fuss> +# RequestHeader append X-Forwarded-Prefix "/fuss" +# RequestHeader set X-Forwarded-Proto "https" +# </Location> +pl = 0 +if "PROXY_LAYERS" in app.config: + pl = app.config["PROXY_LAYERS"] + app.wsgi_app = ProxyFix(app.wsgi_app,x_for=pl,x_host=pl,x_port=pl,x_prefix=pl,x_proto=pl) + +def list_of_files(path=None, meta=False): + if path is None: + path = up + files = [] + try: + files2 = os.listdir(path) + for file in files2: + # never list meta files even if META_VISIBLE + if (not meta and not file.endswith(".meta")) or (meta and file.endswith(".meta")): + files.append(os.path.join(path,file)) + except Exception as E: + return "Problem fetching files from path {0}: {1}".format(path,E) + return files + +def stats(file, lastmodified=-1): + if isinstance(file, FileStorage): + # if file is incoming stream + data = file.stream.read() + if file.content_length > 0: + size = file.content_length + else: + size = len(data) + sha256sum = sha256(data).hexdigest() + if not file.content_type or not "/" in file.content_type or file.content_type == "application/octet-stream": + mimetype = detect_from_buffer(data) + else: + mimetype = file.content_type + if int(lastmodified) > 0: + ts = lastmodified + else: + ts = now() + return file.filename, size, ts, sha256sum, mimetype + else: + # so far, assume it is a regular os PathLike file or string + name = os.path.basename(file) + stat = os.stat(file) + size = stat.st_size + with open(file,"rb") as f: + sha256sum = sha256(f.read()).hexdigest() + ts = int(stat.st_mtime) + mimetype = detect_from_file(file) + return name, size, ts, sha256sum, mimetype + if False: + return str(file), 0, 0, 0, "plain/unknown" + +def pprint(file,format="text",cr=False,file_app_prefix=aps_ap_dlp): + message = "" + if format not in app.config["ACCEPTED_FORMATS"]: + app.logger.warn("format {0} is not in {1}".format(format,accepted_formats)) + format = "text" + name, size, ts, sha256sum, mimetype = stats(file) + if app.config["META"]: + metadata = meta(file) + if format == "text": + message = "{0},{1},{2},{3},{4}".format(name,size,ts,sha256sum,mimetype) + ("\n" if cr else "") + elif format == "html": + message = '<a href="{0}{1}">{1}</a>'.format(aps_ap_dlp,name) + ("<br/>\n" if cr else "") + elif format == "html-long": + message += "<tr>" + for word in [name,size,ts,sha256sum,mimetype]: + if word != name: + message += "<td>{0}</td>".format(word) + else: + if app.config["META"] and app.config["ICONS"]: + icon = "" + try: + icon = metadata["icondata"] + except: + pass + message += '<td><a href="{0}{1}"><img src="{2}">{1}</a></td>'.format(file_app_prefix,word,icon) + else: + message += '<td><a href="{0}{1}">{1}</a></td>'.format(file_app_prefix,word) + message += "</tr>\n" + elif format == "json": + message = { + 'name': name, + 'size': size, + 'timestamp': ts, + 'sha256sum': sha256sum, + 'mimetype': mimetype + } + return message + +def print_files(path=None,format="text",header=False): + result = "" + files = list_of_files(path) + if format == "json": + fl = [] + for file in files: + fl.append(pprint(file,format="json",cr=False)) + result = json.dumps({ 'files': fl }) + elif format == "html": + for file in files: + result += pprint(file,format=format,cr=True) + elif format == "html-long": + _s_ap_s = s_ap_s + _aps_ap_dlp = aps_ap_dlp + # discover how the app was accessed, if configured + _server = aps + _prefix = ap + if "DISCOVER_URL" in app.config and app.config["DISCOVER_URL"]: + try: + _server = request.headers["X-Forwarded-Proto"] + "://" + request.headers["X-Forwarded-Host"] + _temp = _server.replace(", ", "\n").split('\n') + if len(_temp) > 1: + _server = _temp[0] + #print("Customized _server {0}".format(_server)) + except: + _server = "http://" + request.headers["Host"] + try: + _prefix = request.headers["X-Forwarded-Prefix"] + _temp = _prefix.replace(", ", "\n").split('\n') + if len(_temp) > 1: + _prefix = _temp[0] + except: + _prefix = "/" + app.logger.warn("Failed to get x-forwarded-prefix") + _prefix = str("/" + _prefix.strip("/") + "/").replace("//","/") + _aps_ap_dlp = _server.strip("/") + str(_prefix + dlp.strip("/") + "/").replace("//","/") + app.logger.info("Using customized from headers, aps_ap_dlp: {0}".format(_aps_ap_dlp)) + result += "<html>\n<head>\n<link rel=\"stylesheet\" href=\"{0}static/files.css\"/>".format(_prefix) + result += "<table>\n" + if header: + result += "<tr>" + for word in ['link','size','timestamp','sha256sum','mimetype']: + result += "<th>{0}</th>".format(word) + result += "</tr>\n" + for file in files: + result += pprint(file,format=format,cr=True,file_app_prefix=_aps_ap_dlp) + result += "</table>" + else: + # assume "text" + if header: + result += "# {0},{1},{2},{3},{4}\n".format( + "filename", + "size", + "timestamp", + "sha256sum", + "mimetype" + ) + for file in files: + result += pprint(file,cr=True) + + #print("HEADERS for request-files:") + ##for i in dict(request.headers): + # print("HEAD: {0}: {1}".format(i,request.headers[i])) + return result + +def print_config(app,format="text",header=False): + result = "" + if format == "json": + fl = {} + for item in app.config: + if item in app.config["VISIBLE_CONFIGS"]: + fl[item]=app.config[item] + result = json.dumps({'config': fl}) + else: + # assume text + for item in app.config: + if item in app.config["VISIBLE_CONFIGS"]: + result += str(item) + "=" + str(app.config[item]) + "\n" + + return result + +# will only be called if META=true +def meta(file, variable=None, value=None, debug=False): + #print("META WAS CALLED! '{0}' '{1}' '{2}'".format(file,variable, value)) + metafile = os.path.join(os.path.dirname(file), + "."+os.path.basename(file)+".meta" + ) + metadata = {} + try: + try: + with open(metafile,"r") as mf: + metadata = json.loads(mf.read()) + except: + # probably no file exists + pass + # if writing to the metadata file + if value: + metadata[variable] = value + if debug: + #print("META DEBUG assigning {0} = {1}".format(variable,value)) + #print("META DEBUG metafile {0}".format(metafile)) + app.logger.info(json.dumps(metadata)) + with open(metafile,"w") as mf: + json.dump(metadata,mf) + if debug: + for i in metadata: + app.logger.info("META DEBUG i={0} value={1}".format(i,metadata[i])) + except Exception as E: + app.logger.error("Error happened: {0}".format(E)) + pass + # return a dictionary of the values from the meta file + if variable and variable in metadata: + return metadata[variable] + + # otherwise, return the dictionary + return metadata + +def store_file(file, lastmodified=-1,addr=None): + skip_write = False + name, size, ts, sha256sum, mimetype = stats(file, lastmodified) + try: + # reset stream so I can read it again here + file.seek(0) + except: + pass + data = file.stream.read() + app.logger.info(name + " " + str(size) + " " + str(ts) + " " + str(sha256sum) + " " + mimetype) + + # Ensure mimetype is allowed + if mimetype in app.config["MIMETYPE_BLACKLIST"]: + app.logger.info("Problem with filetype.") + return "Mimetype {0} is disallowed.\n".format(mimetype), 415 + + # WHITELIST functionality would go here if implemented. + + # Verify it is a safe filename + # manually check for '-' which secure_filename thinks is OK but it is not. + if name == '-': + # then the content was piped in to curl -F 'file=@-' + name = "stdin" + dest_filename = secure_filename(name) + dest_file = os.path.join(up, dest_filename) + + if os.path.exists(dest_file): + ename, esize, ets, esum, emime = stats(dest_file) + if esum == sha256sum: + skip_write = True + app.logger.info("Existing file {0} is same as uploaded file.".format(ename)) + else: + x = 0 + safe = False + # if file is not identical, increment until it is a new number and then save + while not safe: + x += 1 + if x >= app.config["MAX_DUPLICATE_NAMES"]: + app.logger.error("Have to stop trying now.") + return "Too many versions already exist for {0}\n".format(dest_filename), 409 + + name_array = dest_filename.split(".") + ext = "" + if len(name_array) > 1: + name_array[-2] = name_array[-2] + "-" + str(x) + temp_dest_filename = '.'.join(name_array) + else: + temp_dest_filename = dest_filename + "-" + str(x) + + temp_dest_file = os.path.join(up, temp_dest_filename) + if os.path.exists(temp_dest_file): + ename, esize, ets, esum, emime = stats(temp_dest_file) + # if file is identical, continue silently + if esum == sha256sum: + safe = True + skip_write = True + app.logger.info("Existing file {0} is same as uploaded file.".format(ename)) + dest_filename = temp_dest_filename + dest_file = temp_dest_file + else: + safe = True + app.logger.info("Found safe filename {0}".format(temp_dest_file)) + dest_filename = temp_dest_filename + dest_file = temp_dest_file + #else: + + if not skip_write: + if size > app.config["MAX_FILE_SIZE"]: + return "File size {0} is too large for limit {1}\n".format(size,app.config["MAX_FILE_SIZE"]), 413 + with open(dest_file, "wb") as f: + f.write(data) + # write timestamp + ts = int(ts) + if ts > 0: + os.utime(dest_file, (ts,ts)) + + if app.config["META"]: + meta(dest_file,"address",addr) + meta(dest_file,"uploaded",now()) + app.logger.info("SAVED TO " + dest_file) + + return url_for("get",filename=dest_filename, _external=True) + "\n", 301 if skip_write else 201 + +def html_template(filename="",full=False): + # discover how the app was accessed, if configured + _server = aps + _prefix = ap + if "DISCOVER_URL" in app.config and app.config["DISCOVER_URL"]: + try: + _server = request.headers["X-Forwarded-Proto"] + "://" + request.headers["X-Forwarded-Host"] + _temp = _server.replace(", ", "\n").split('\n') + if len(_temp) > 1: + _server = _temp[0] + #print("Customized _server {0}".format(_server)) + except: + _server = "http://" + request.headers["Host"] + try: + _prefix = request.headers["X-Forwarded-Prefix"] + _temp = _prefix.replace(", ", "\n").split('\n') + if len(_temp) > 1: + _prefix = _temp[0] + except: + _prefix = "/" + if full: + # takes extra processing + max_size = app.config["MAX_FILE_SIZE"] + size_suffix = "B" + if max_size > 1024 * 10: + max_size = max_size/1024 + size_suffix = "KB" + if max_size > 1024 * 10: + max_size = max_size/1024 + size_suffix = "MB" + if max_size > 1024 * 3: + max_size = max_size/1024 + size_suffix = "GB" + if size_suffix == "GB": + max_size = round(max_size, 2) + else: + max_size = int(max_size) + meta = False + meta_visible = False + meta_headers = [] + icons = False + try: + meta = app.config["META"] + except: + pass + try: + meta_visible = app.config["META_VISIBLE"] + except: + pass + try: + meta_headers = app.config["META_HEADERS"] + except: + pass + try: + icons = app.config["ICONS"] + except: + pass + return render_template(filename + ".j2", + server = _server, + prefix = _prefix, + ulp = ulp, + file_count = len(list_of_files()), + max_size = str(max_size) + size_suffix, + mimetype_blacklist = app.config["MIMETYPE_BLACKLIST"], + max_dupe = app.config["MAX_DUPLICATE_NAMES"], + meta = meta, + meta_visible = meta_visible, + meta_headers = meta_headers, + icons = icons + ) + else: + #print("was asked for {0}".format(filename)) + return render_template(filename + ".j2", + server = _server, + prefix = _prefix, + ulp = ulp + ) + +@timer(app.config["LOOP_DELAY"]) +def loop(num): + lof = list_of_files(meta=True) + app.logger.info("Loop starts {0}".format(datetime.today())) + # Task 1: clean up meta files for files that no longer exist + for f in lof: + nonmetafile=os.path.join(os.path.dirname(f),os.path.basename(f).lstrip(".").rstrip(".meta")) + if not os.path.exists(nonmetafile): + try: + os.remove(f) + except: + app.logger.error("Please fix the @timer LOOP_DELAY") + #print("Process {0}".format(nonmetafile)) + # must regenerate list becauase we might have shortened it + + # Task 2: add icon if undefined + # FUTURE IMPROVEMENT: PERFORMANCE needs to be improved. For some reason, some meta files have "icondata" values but these values are not recognized by the dict lookup, so it looks up their icontype every single run. + if "ICONS" in app.config and app.config["ICONS"]: + lof = list_of_files(meta=False) + if "ICON_THEME" in app.config: + # this seems to fail silently and just load "hicolor" or something basic, which is good enough. + Gtk.Settings.get_default().set_property('gtk-icon-theme-name',app.config["ICON_THEME"]) + icon_theme = Gtk.IconTheme.get_default() + for f in lof: + name, _, _, _, mimetype = stats(f) + fmetadata = meta(f) + fm_trim = trim_dict(fmetadata) + app.logger.info("FOR FILE " + name) + app.logger.info("found metadata " + str(fm_trim)) + app.logger.info("SHOWING ITEMS IN fmetadata length {1} for {0}".format(f,len(fmetadata))) + #for item in fmetadata: + # print("item: {0} value: {1}".format(item,fmetadata[item])) + #_ = fmetadata["icondata"] + try: + _ = fmetadata["icondata"] + except: + app.logger.info("NEED to add icondata for {0}".format(name)) + icon = Gio.content_type_get_icon(mimetype) + image_file = None + image_base64 = None + for entry in icon.to_string().split(): + if entry != "." and entry != "GThemedIcon": + #print("Need to check entry {0}".format(entry)) + try: + image_file = icon_theme.lookup_icon(entry,32,0).get_filename() + except: + # this entry in the list must not have an image + pass + if image_file: + break + #print("Found icon {0}".format(icon.to_string())) + #icon_file = icon_theme.lookup_icon(icon.to_string(), 48, 0) + #print("file {0} is type {1}, icon {2}".format(name,mimetype,image_file)) + if image_file: + app.logger.info("Found for {0} icon {1}".format(name,image_file)) + # load data from file, save as base64-encoded image + # by following symlink, we can avoid mimetype of "inode/symlink" + followed_link = os.path.realpath(image_file) + _, _, _, _, image_mime = stats(followed_link) + with open(image_file,"rb") as i: + raw = i.read() + image_base64 = "data:{0};base64,".format(image_mime) + str(base64.b64encode(raw).decode('utf-8')) + meta(f,"icondata",image_base64,debug=True) + #print("file {0} is type {1}, icondata {2}".format(name,mimetype,image_base64)) + app.logger.info("Found for {0} file {1}".format(name,image_file)) + # END for f in lof. + +@app.route("/") +def root(): + return html_template("index.html",True) + +@app.route("/robots.txt") +def robots(): + return send_from_directory("static","robots.txt") + +@app.route("/fuss-upload") +def fuss_upload(): + return send_from_directory('/usr/share/fuss/static',"fuss-upload",mimetype="application/x-shellscript") + +@app.route("/dump_files/") +@app.route("/dump_files/<format>") +def show_files(format="text"): + if format not in mime_dict: + format = "text" + return Response(print_files(up,format=format, header=True), mimetype=mime_dict[format]) + +@app.route("/dump_config/") +@app.route("/dump_config/<format>") +def show_config(format="text"): + #def gen(): + # for item in app.config: + # if item in app.config["VISIBLE_CONFIGS"]: + # yield str(item) + "=" + str(app.config[item]) + "\n" + #return Response(gen(), mimetype="text/plain") + if format not in mime_dict: + format = "text" + return Response(print_config(app,format=format, header=True), mimetype=mime_dict[format]) + +# Upload route +@app.route("/" + ulp_s, methods=["GET","POST"]) +def upload(): + message = "" + headers = None + if request.headers: + headers = dict(request.headers) + if request.method == "POST": + # debug + #for h in headers: + # print("{0}: \"{1}\"".format(h,headers[h])) + #print("END OF HEADERS") + if "DISCOVER_URL" in app.config and app.config["DISCOVER_URL"]: + if "X-Forwarded-For" in headers: + pl = len(headers["X-Forwarded-Host"].split(", ")) + app.wsgi_app = ProxyFix(app.wsgi_app,x_for=pl,x_host=pl,x_port=pl,x_prefix=pl,x_proto=pl) + response_codes = [] + if request.files: + # for a single file upload: + #infile = request.files["file"] + f_count = -1 + for f in request.files.getlist("file"): + f_count += 1 + infile = f + lastModified = -1 + + # the html upload form passes in a cookie + if request.cookies: + if "lastmod" in request.cookies: + lastModified = int(request.cookies["lastmod"].split(",")[f_count]) + # Loop through cookies with this + #for c in request.cookies: + # print("Deal with {0} which is {1}".format(c,request.cookies[c])) + + if headers: + if 'lastModified' in headers: + try: + lastModified = headers["lastModified"] + except: + pass + ip_addr = request.remote_addr + if "HTTP_X_FORWARDED_FOR" in request.environ: + app.logger.info("Found x-forwarded-for= " + request.environ["HTTP_X_FORWARDED_FOR"]) + ip_addr = request.environ["HTTP_X_FORWARDED_FOR"] + store_result = store_file(infile, lastmodified=lastModified,addr=ip_addr) + message += str(store_result).split("'")[1].replace("\\n","<br/>") + _, foo = store_result + response_codes.append(foo) + elif request.method == "GET": + if "DISCOVER_URL" in app.config and app.config["DISCOVER_URL"]: + if "X-Forwarded-For" in headers: + pl = len(headers["X-Forwarded-Host"].split(", ")) + app.wsgi_app = ProxyFix(app.wsgi_app,x_for=pl,x_host=pl,x_port=pl,x_prefix=pl,x_proto=pl) + return html_template("upload.html") + + # Have to figure the best response, when handling multiple file uploads + #print("List of result codes:",response_codes) + final_response_code = 400 + if len(response_codes) == 1 or all_same(response_codes): + final_response_code = response_codes[0] + elif "415" in response_codes: + final_response_code = 415 + elif "409" in response_codes: + final_response_code = 409 + elif "413" in response_codes: + final_response_code = 413 + elif "301" in response_codes: + final_response_code = 301 + else: + final_response_code = 201 + return message, final_response_code + +@app.route("/template/<filename>") +def get_template(filename): + return html_template(filename) + +# Download route +@app.route("/" + dlp.strip("/") + "/<filename>") +def get(filename): + # safety: remove leading dots + filename = filename.lstrip(".") + + if filename.endswith(".meta"): + if app.config["META"] and app.config["META_VISIBLE"]: + # the metadata file stored in disk has a filename that starts with a dot + filename = "." + filename + else: + # we are not allowed to dispense metadata + return redirect(url_for("get",filename=filename.replace(".meta",""), _external=True)) + + file = os.path.join(up,filename) + result = "" + app.logger.info("Dealing with {0}".format(file)) + if not os.path.exists(file): + abort(404) + try: + name, size, ts, sha256sum, mimetype = stats(file) + except: + result += "Unable to get size of file {0}\n".format(file) + # this should not be here?! + #metadata = meta(file) + if result != "": + return result + # Nginx trick for telling web server to handle the file instead of having this application do it + if app.config["USE_X_ACCEL_REDIRECT"]: + response = make_response() + response.headers["Content-Type"] = mimetype + response.headers["Content-Length"] = size + response.headers["X-Accel-Redirect"] = "/" + os.path.join(up, name) + else: + response = send_from_directory(up, name, mimetype = mimetype) + response.headers['Digest'] = "sha-256={0}".format(sha256sum) + + if app.config["META"] and app.config["META_VISIBLE"]and "META_HEADERS" in app.config: + metadata = meta(file) + fm_trim = trim_dict(metadata) + app.logger.info("found metadata " + str(fm_trim)) + for item in metadata: + if item in app.config["META_HEADERS"]: + response.headers[str("X-meta-"+item)] = metadata[item] + + return response + +# Set logging level here, and other useful values +app.logger.setLevel("INFO") +app.static_folder=app.config["STATIC_FOLDER"] +app.template_folder=app.config["TEMPLATE_FOLDER"] + +if __name__ == "__main__": + manager.add_command('runserver', Server(host=app.config["APP_HOST"], port=app.config["APP_PORT"])) + app.run() +# vim: set sw=3 ts=3 sts=3 et: diff --git a/fuss.wsgi.ini.example b/fuss.wsgi.ini.example new file mode 100644 index 0000000..173173f --- /dev/null +++ b/fuss.wsgi.ini.example @@ -0,0 +1,18 @@ +# call with: uwsgi_python3 --ini fuss.wsgi.ini +[uwsgi] +plugins = python3,logfile +http-socket = 0.0.0.0:5003 +wsgi-file = /usr/libexec/fuss/fuss.py +callable = app +touch-reload = /usr/libexec/fuss/fuss.py +touch-reload = /etc/fuss.conf +touch-reload = /etc/fuss.wsgi.ini +touch-reload = /usr/share/fuss/templates/upload.html.j2 +touch-reload = /usr/share/fuss/templates/upload.js.j2 +touch-reload = /usr/share/fuss/templates/index.html.j2 +req-logger = file:/var/log/fuss/req.log +# to get strftime format fields, you need double percent signs +logdate = "%%FT%%T" +logger = file:/var/log/fuss/fuss.log +# the init script uses a different pidfile owned by root. +pidfile = /var/www/fuss/fuss.pid diff --git a/static/files.css b/static/files.css new file mode 100644 index 0000000..2668fca --- /dev/null +++ b/static/files.css @@ -0,0 +1,4 @@ +img { + max-width: 32px; + max-height: 32px; +} diff --git a/static/fuss-upload b/static/fuss-upload new file mode 100755 index 0000000..6d23b25 --- /dev/null +++ b/static/fuss-upload @@ -0,0 +1,8 @@ +#!/bin/sh +# File: fuss-upload.sh +# Goal: wrap around curl to include the timestamp header to fuss +test -z "${INFILE}" && INFILE="${1}" +test -z "${FUSS_URL}" && FUSS_URL="${2}" +test ! -r "${INFILE}" && { echo "Cannot read file ${INFILE}. Aborted." 1>&2 ; exit 1; } +lastModified="$( stat -c "%Y" "${INFILE}" )" +curl -v --header "lastModified: ${lastModified}" -F "file=@${INFILE}" "${FUSS_URL}" diff --git a/static/index.css b/static/index.css new file mode 100644 index 0000000..fd8d87b --- /dev/null +++ b/static/index.css @@ -0,0 +1,104 @@ +:root { + --heading: #FFBA80; + --heading-mobile: #FF9A80; + --heading-hover: #FFD0A0; + --radius: 5px; + --radius-mobile: 2px; + --pre-background: rgba(0, 0, 0, .15); +} + +input[type='checkbox'] { + display: none; +} + +@viewport { + width: device-width ; + zoom: 1.0 ; +} + +.lbl-toggle { + display: block; + background: var(--heading); + cursor: pointer; + border-radius: var(--radius); + /* resemble h2 */ + font-weight: bold; + font-size: x-large; +} + +.lbl-toggle:hover { + background-color: var(--heading-hover); +} + +/* add the triangle */ +.lbl-toggle::before { + content: ' '; + display: inline-block; + + border-top: 5px solid transparent; + border-bottom: 5px solid transparent; + border-left: 5px solid currentColor; + + vertical-align: middle; + margin-right: .7rem; + transform: translateY(-2px); + + transition: transform .2s ease-out; +} + +.content .content-inner { + background: rgba(200, 180, 128, .2); + border-bottom: 1px solid var(--heading); + + border-bottom-left-radius: var(--radius); + border-bottom-right-radius: var(--radius); + padding: .5rem 1rem; +} + +.content { + max-height: 0px; + overflow: hidden; + + transition: max-height .25s ease-in-out; +} + +.toggle:checked + .lbl-toggle + .content { + max-height: 100vh; +} +.toggle:checked + .lbl-toggle::before { + transform: rotate(90deg) translateX(-3px); +} + +.toggle:checked + .lbl-toggle { + border-bottom-right-radius: 0; + border-bottom-left-radius: 0; +} + +.content pre { + background: var(--pre-background); + border: 1px solid; +} + +/* For narrow screens (probably mobile devices) + * which works because html includes meta viewport. + */ +@media screen and (max-width: 500px) { + + .lbl-toggle { + background: var(--heading-mobile); + border-radius: var(--radius-mobile); + font-weight: normal; + } + .content pre { + /* wrap long text and urls */ + white-space: pre; /* CSS 2.0 */ + white-space: pre-wrap; /* CSS 2.1 */ + white-space: pre-line; /* CSS 3.0 */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: -moz-pre-wrap; /* Mozilla */ + word-wrap: break-word; /* IE 5+ */ + } + +} +/* vim: set ts=2 sw=2 sts=2 et: */ diff --git a/static/index.js b/static/index.js new file mode 100644 index 0000000..291c256 --- /dev/null +++ b/static/index.js @@ -0,0 +1,25 @@ +/* Allow toggling the checkboxes with space, enter, and left/right */ +let myLabels = document.querySelectorAll('.lbl-toggle'); +Array.from(myLabels).forEach(label => { + label.addEventListener('keydown', e => { + // 32 spacebar, 13 enter + if (e.which === 32 || e.which === 13) { + e.preventDefault(); label.click(); + }; + // 37 left, 39 right + if (e.which === 37 || e.which === 39) { + var inputs = document.getElementsByTagName('INPUT'); + for (var i=0; i < inputs.length; i++) { + if (inputs[i].id === e.target.htmlFor) { + if (!inputs[i].checked && e.which === 39) { + e.preventDefault(); label.click(); + } + if (inputs[i].checked && e.which === 37) { + e.preventDefault(); label.click(); + } + } + } + } + }); +}); + diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/static/upload.css b/static/upload.css new file mode 100644 index 0000000..0b37238 --- /dev/null +++ b/static/upload.css @@ -0,0 +1,72 @@ +/* (A) UPLOAD ZONE */ +/* vim:set ts=2 sw=2 sts=2 et: */ + +:root { + --blue: #cfd5ff; + --radius: 5px; + --yellow: #f9fcbd; + --green: #bdfcbe; + --red: #ffcfd5; + --boxshadow1: inset 0px 0px 0px 8px; + --boxshadow2: 0px 0px 0px 8px; +} + +#upzone { + width: 300px; + height: 200px; + background: var(--blue); + padding: 0px; + text-align: center; + display: table-cell; + vertical-align: middle; + /* border: 5px solid var(--blue); */ + ; + box-shadow: var(--boxshadow1) var(--blue), + var(--boxshadow2) var(--blue); +} + +/* when hovering over it with a file */ +#upzone.highlight { + background: var(--green); + box-shadow: var(--boxshadow1) var(--green), + var(--boxshadow2) var(--green); +} + +/* when uploading files in the queue */ +#upzone.processing { + background: var(--yellow); + box-shadow: var(--boxshadow1) var(--yellow), + var(--boxshadow2) var(--yellow); +} + +#upzone.invalid { + background: var(--red); + box-shadow: var(--boxshadow1) var(--red), + var(--boxshadow2) var(--red); +} + +#upzone.processing, #upzone.highlight, #upzone.invalid, #upzone { + /* nice transition effect when beginning to upload */ + -webkit-transition: all 0.25s linear; + -moz-transition: all 0.25s linear; + -o-transition: all 0.25s linear; + transition: all 0.25s linear; + border: 3px solid rgba(0,0,0,0.1); + border-radius: var(--radius); + border-style: dashed; +} + +/* (B) UPLOAD FORM */ +#upform { + /* display anyway, even with upload zone visible */ + display: block; +} + +#upstat { + font-size: 70%; +} + +/* this div will be set to display: block by javascript */ +#hiddenfirst { + display: none; +} diff --git a/templates/index.html.j2 b/templates/index.html.j2 new file mode 100644 index 0000000..75f97ee --- /dev/null +++ b/templates/index.html.j2 @@ -0,0 +1,83 @@ +{% set prefix_s = prefix | trim('/') ~ '/' %}{% set s_prefix_s = ( '/' ~ prefix | trim('/') ~ '/' ) | replace ("//","/") %}{% set ulp_s = ulp | trim('/') ~ '/' %}{% set server_prefix = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %}{% set prefix_ulp = ( '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") %}{% set server_prefix_ulp = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %}<!DOCTYPE html> +<html> +<head> +<title>FUSS: File Upload and Storage Service</title> +<link rel="stylesheet" href="{{ s_prefix_s }}static/index.css"/> +<meta name="viewport" content="width=device-width, initial-scale=1"> +</head> +<!-- +server = "{{ server }}" +prefix = "{{ prefix }}" +ulp = "{{ ulp }}" +file_count = "{{ file_count }}" +max_size = "{{ max_size }}" +mimetype_blacklist = "{{ mimetype_blacklist }}" +meta = "{{ meta }}" +meta_visible = "{{ meta_visible }}" +meta_headers = "{{ meta_headers }}" +icons = "{{ icons }}" +--> +<body> +<h1>FUSS</h1> +<input id="intro" class="toggle" type="checkbox" checked><label for="intro" class="lbl-toggle" tabindex="0">Introduction</label> +<div id="intro" class="content"> +<div class="content-inner"> +<a href="{{ server_prefix }}">FUSS</a> is a simple file upload and storage web app. +Features include: +<ul> +<li>Maximum upload size</li> +<li>Accept original file modification timestamp</li> +<li>Filter out mimetypes</li> +<li>Handle duplicate files gracefully</li> +<li>Handle up to so many dissimilar files with same name</li> +<li>Drag and drop upload</li> +<li>Display mimetype icons</li> +</ul> +<p style="font-size: 66%;"><a href="https://gitlab.com/bgstack15/fuss/">source</a> (GPL-3.0)</p> +</div> <!-- content-inner --> +</div> <!-- content --> +<input id="use" class="toggle" type="checkbox" checked><label for="use" class="lbl-toggle" tabindex="0">How to use</label> +<div id="use" class="content"> +<div class="content-inner"> +{% if ulp != "" %}Visit the upload form at <a href="{{ prefix_ulp }}">{{ ulp_s }}</a>. +Or f{% else %}F{% endif %}rom the command line: +<pre> +curl -F 'file=@/path/to/file' {{ server_prefix_ulp }} +</pre> +To save the original last modified timestamp from the command line, use script <a href="{{ s_prefix_s }}fuss-upload">fuss-upload</a>. +<pre> +./fuss-upload /path/to/file {{ server_prefix_ulp }} +</pre> +</div> <!-- content-inner --> +</div> <!-- content --> +<input id="stats" class="toggle" type="checkbox" checked><label for="stats" class="lbl-toggle" tabindex="0">Current status for this instance</label> +<div id="stats" class="content"> +<div class="content-inner"> +<ul> +<li>Max file size: {{ max_size }}</li> +<li>Files stored: <a href="{{ s_prefix_s }}dump_files/">{{ file_count }}</a> +(<a href="{{ s_prefix_s }}dump_files/json">json</a> +<a href="{{ s_prefix_s }}dump_files/html">html</a> +<a href="{{ s_prefix_s }}dump_files/html-long">html details</a>) +</li> +<li>Mimetype blacklist +{% if mimetype_blacklist %} +<ul>{% endif %} +{% for m in mimetype_blacklist %}<li>{{ m }}</li>{% endfor %} +{% if mimetype_blacklist %}</ul>{% endif %} +<li>Max duplicate filenames: {{ max_dupe }}</li> +<li>Metadata{% if not meta %} is disabled{% endif %} +{% if meta %} +<ul> +<li>User queriable: {% if meta_visible %}yes{% else %}no{% endif %}</li> +<li>Add to headers: {% if meta_visible and (meta_headers|length) > 0 %}yes{% else %}no{% endif %}</li> +<li>Mimetype icons: {% if icons %}yes{% else %}no{% endif %}</li> +{% endif %} +</ul> <!-- metadata sublist --> +</ul> +<p style="font-size: 66%;"><a href="{{ s_prefix_s }}dump_config/">dump config</a> (<a href="{{ s_prefix_s }}dump_config/json">json</a>) +</div> <!-- content-inner --> +</div> <!-- content --> +<script src="{{ s_prefix_s }}static/index.js"></script> +</body> +</html> diff --git a/templates/upload.html.j2 b/templates/upload.html.j2 new file mode 100644 index 0000000..35673b7 --- /dev/null +++ b/templates/upload.html.j2 @@ -0,0 +1,53 @@ +{% set prefix_s = prefix | trim('/') ~ '/' %}{% set s_prefix_s = ( '/' ~ prefix | trim('/') ~ '/' ) | replace ("//","/") %}{% set ulp_s = ulp | trim('/') ~ '/' %}{% set server_prefix = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %}{% set prefix_ulp = ( '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") %}{% set server_prefix_ulp = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %}<!DOCTYPE html> +<html> + <head> + <title> + FUSS upload page + </title> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <!-- (A) CSS + JS --> + <link rel="stylesheet" href="{{ s_prefix_s }}static/upload.css"/> + <link rel="stylesheet" href="{{ s_prefix_s }}static/index.css"/> + <script src="{{ s_prefix_s }}template/upload.js"></script> + <script> + function lastmod(elem){ + tc = "lastmod="; + for (var x=0; x < elem.files.length; x++) { + tc += `${Math.trunc(elem.files[x].lastModified / 1000)},` + } + document.cookie = tc; + // For a single file: + //document.cookie = `lastmod=${elem.files[0].lastModified}` + } + </script> + </head> +<!-- +server = "{{ server }}" +prefix = "{{ prefix }}" +ulp = "{{ ulp }}" +--> + <body> + <!-- (B) FILE DROP ZONE --> + <div id="upzone"> + Drop Files Here + </div> + + <!-- (D) FALLBACK --> + <form id="upform" action="{{ server_prefix_ulp }}" method="post" enctype="multipart/form-data"> + <br/> + <input type="file" name="file" multiple accept="*" oninput="lastmod(this);" required> + <input type="submit" value="Upload File"> + </form> + + <!-- (C) UPLOAD STATUS --> + <div id="hiddenfirst"> + <input id="upstatus" class="toggle" type="checkbox" checked><label for="upstatus" class="lbl-toggle" tabindex="0">Upload status</label> + <div id="upstatus" class="content"> + <div class="content-inner" id="upstat"> + </div> <!-- content-inner --> + </div> <!-- content --> + </div> + <script src="{{ s_prefix_s }}static/index.js"></script> + </body> +</html> diff --git a/templates/upload.js.j2 b/templates/upload.js.j2 new file mode 100644 index 0000000..28fe892 --- /dev/null +++ b/templates/upload.js.j2 @@ -0,0 +1,139 @@ +{% set prefix_s = prefix | trim('/') ~ '/' %}{% set s_prefix_s = ( '/' ~ prefix | trim('/') ~ '/' ) | replace ("//","/") %}{% set ulp_s = ulp | trim('/') ~ '/' %}{% set server_prefix = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %}{% set prefix_ulp = ( '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") %}{% set server_prefix_ulp = ( server | replace("//","##") ~ '/' ~ prefix ~ '/' ~ ulp ~ '/' ) | replace("///","/") | replace("//", "/") | replace("##","//") %} +/* vim:set syntax=javascript ts=2 sw=2 sts=2 et: */ +var ddup = { + // (A) ON PAGE LOAD + hzone: null, // HTML upload zone + hstat: null, // HTML upload status + hform: null, // HTML upload form + hiddenfirst: null, + init : function () { + // (A1) GET HTML ELEMENTS + ddup.hzone = document.getElementById("upzone"); + ddup.hstat = document.getElementById("upstat"); + ddup.hform = document.getElementById("upform"); + ddup.hiddenfirst = document.getElementById("hiddenfirst"); + + // (A2) DRAG-DROP SUPPORTED + if (window.File && window.FileReader && window.FileList && window.Blob) { + // HIGHLIGHT DROPZONE ON FILE HOVER + ddup.hzone.addEventListener("dragenter", function (e) { + e.preventDefault(); + e.stopPropagation(); + ddup.hzone.classList.add('highlight'); + ddup.hzone.innerHTML = "Release to upload"; + }); + ddup.hzone.addEventListener("dragleave", function (e) { + e.preventDefault(); + e.stopPropagation(); + ddup.hzone.classList.remove('highlight'); + ddup.hzone.innerHTML = "Drop Files Here"; + }); + + // DROP TO UPLOAD FILE + ddup.hzone.addEventListener("dragover", function (e) { + e.preventDefault(); + e.stopPropagation(); + }); + ddup.hzone.addEventListener("drop", function (e) { + e.preventDefault(); + e.stopPropagation(); + ddup.hzone.classList.remove('highlight'); + ddup.hiddenfirst.style.display = "block" ; + if (e.dataTransfer.files.length > 0) { + ddup.hzone.classList.add('processing'); + ddup.hzone.innerHTML = "Uploading..."; + ddup.queue(e.dataTransfer.files); + } else { + ddup.hzone.classList.add('invalid'); + ddup.hzone.innerHTML = "Input is not recognized!"; + ddup.reset_text_timer(1500); + } + }); + } + + // (A3) DRAG-DROP UPLOAD NOT SUPPORTED + else { + ddup.hzone.style.display = "none"; + ddup.hform.style.display = "block"; + } + }, + + // (B) UPLOAD QUEUE + HANDLER + // NOTE: AJAX IS ASYNCHRONOUS + // A QUEUE IS REQUIRED TO STOP SERVER FLOOD + upqueue : [], // upload queue + uplock : false, // currently uploading a file + queue : function (files) { + // FILE LIST INTO QUEUE + for (let f of files) { + // OPTIONAL - SHOW UPLOAD STATUS + ddup.thisdiv = document.getElementById(f.name); + if (!ddup.thisdiv) { + // add new div with filename as id + ddup.hstat.innerHTML += `<div id="${f.name}">${f.name} - Added to queue</div>`; + } else { + // change contents of existing div + ddup.thisdiv.innerHTML = `${f.name} - Added to queue`; + } + // ADD TO QUEUE + ddup.upqueue.push(f); + } + // GO! + ddup.go(); + }, + + // (C) AJAX UPLOAD + go : function () { if (!ddup.uplock && ddup.upqueue.length!=0) { + // (C1) QUEUE STATUS UPDATE + ddup.uplock = true; + + // (C2) PLUCK OUT FIRST FILE IN QUEUE + let thisfile = ddup.upqueue[0]; + ddup.upqueue.shift(); + + // OPTIONAL - SHOW UPLOAD STATUS + ddup.thisdiv = document.getElementById(thisfile.name); + ddup.thisdiv.innerHTML = `${thisfile.name} - Upload started`; + // at start of queue, change color + ddup.hzone.classList.add('processing'); + ddup.hzone.innerHTML = "Uploading..."; + + // (C3) UPLOAD DATA + let data = new FormData(); + data.append('file', thisfile); + // ADD MORE POST DATA IF YOU WANT + // data.append("KEY", "VALUE"); + + // (C4) AJAX REQUEST + let xhr = new XMLHttpRequest(); + xhr.open("POST", "{{ server_prefix_ulp }}"); + xhr.setRequestHeader("lastModified", thisfile.lastModified/1000); + xhr.onload = function () { + // OPTIONAL - SHOW UPLOAD STATUS + ddup.thisdiv.innerHTML = `${thisfile.name} - ${this.response}`; + // NEXT BETTER PLAYER! + ddup.hzone.classList.remove('processing'); + ddup.hzone.innerHTML = "Drop Files Here"; + ddup.uplock = false; + ddup.go(); + }; + xhr.onerror = function(evt){ + // OPTIONAL - SHOW UPLOAD STATUS + ddup.thisdiv.innerHTML = `${thisfile.name} - AJAX ERROR`; + // NEXT BETTER PLAYER! + ddup.hzone.classList.remove('processing'); + ddup.hzone.innerHTML = "Drop Files Here"; + ddup.uplock = false; + ddup.go(); + }; + xhr.send(data); + } }, + + reset_text_timer : function (delay_ms) { + setTimeout(function (){ + ddup.hzone.classList.remove('invalid'); + ddup.hzone.innerHTML = "Drop Files Here"; + }, delay_ms); + } +}; +window.addEventListener("DOMContentLoaded", ddup.init); |