From 85df4b3dc87003ae738f2676a99b88a83f3ac05a Mon Sep 17 00:00:00 2001 From: "B. Stack" Date: Sun, 13 Feb 2022 21:47:21 -0500 Subject: add expiry, and wsgi usage to support that The flask dev server seems incapable of using the @timer decorator, so we need the whole wsgi implementation, for which I've added a script. Arbitrary expiration options are available in the config file. --- .gitignore | 2 ++ README-bgstack15.md | 6 ++-- config.cfg.tpl | 13 ++++++++ initdb.py | 1 + pastebin.py | 83 ++++++++++++++++++++++++++++++++++++++++++++--- stackbin.bin | 9 +++++ stackbin.wsgi.ini | 14 ++++++++ static/style.css | 1 + templates/admin.html | 6 ++-- templates/new_paste.html | 7 ++-- templates/show_paste.html | 4 ++- 11 files changed, 134 insertions(+), 12 deletions(-) mode change 100644 => 100755 pastebin.py create mode 100755 stackbin.bin create mode 100644 stackbin.wsgi.ini diff --git a/.gitignore b/.gitignore index abccb3f..a2e3e4d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ env pastebin_uwsgi.ini config.cfg .*.swp +log/ +*.pid diff --git a/README-bgstack15.md b/README-bgstack15.md index e0cd9bc..67b143e 100644 --- a/README-bgstack15.md +++ b/README-bgstack15.md @@ -24,17 +24,19 @@ Run server in development mode. FLASK_APP=pastebin.py FLASK_DEBUG=True flask run --host='0.0.0.0' +Run the server in a full wsgi environment for the cleanup timer to operate. + + ./stackbin.bin + # Improvements I still need to work on these tasks: ## Development * Protect the /admin/ page -* Add expiry of pastes? (use existing pubdate value, or just an extra column with desired expiration timestamp) ## Release -* Read any of my flask projects (fuss is the best one) to learn how to setup prod server * Document centos7 dependencies * Deploy to prod diff --git a/config.cfg.tpl b/config.cfg.tpl index 50b4419..23fb072 100644 --- a/config.cfg.tpl +++ b/config.cfg.tpl @@ -2,3 +2,16 @@ DEBUG=False SQLALCHEMY_DATABASE_URI='sqlite:///pastebin.db' SECRET_KEY='development-key' SALT='jackson' +DELETESALT='differentstring' +APPNAME='stackbin' +# LOOP_DELAY in seconds is how many seconds between running the expiration cleanup task +LOOP_DELAY = 5 * 3 + +# Disable this variable entirely, to disable any choices for expiration +# Any very simple expression for relative time can be used here. This will be processed by pyparsedate. +EXPIRATION_OPTIONS = [ + "never", + "1 day", + "1 hour", + "15 minutes" +] diff --git a/initdb.py b/initdb.py index b56872f..da4f05a 100644 --- a/initdb.py +++ b/initdb.py @@ -1,2 +1,3 @@ from pastebin import db +no_wsgi = True db.create_all() diff --git a/pastebin.py b/pastebin.py old mode 100644 new mode 100755 index 3851d01..d06dc6e --- a/pastebin.py +++ b/pastebin.py @@ -1,7 +1,14 @@ -from datetime import datetime +from datetime import datetime, timedelta from itsdangerous import Signer from flask import (Flask, request, url_for, redirect, g, render_template, session, abort) from flask_sqlalchemy import SQLAlchemy +from pytimeparse.timeparse import timeparse # python3-pytimeparse +# uwsgidecorators load will fail when using initdb.py but is also not necessary +try: + from uwsgidecorators import timer # python3-uwsgidecorators +except: + pass +import time ## ripped from https://stackoverflow.com/questions/183042/how-can-i-use-uuids-in-sqlalchemy/812363#812363 from sqlalchemy import types @@ -12,6 +19,7 @@ class UUID(types.TypeDecorator): impl = MSBinary def __init__(self): self.impl.length = 16 + self.cache_ok = False # to shut up some useless warning types.TypeDecorator.__init__(self,length=self.impl.length) def process_bind_param(self,value,dialect=None): if value and isinstance(value,uuid.UUID): @@ -66,17 +74,27 @@ class Paste(db.Model): code = db.Column(db.Text) title = db.Column(db.Text) pub_date = db.Column(db.DateTime) + exp_date = db.Column(db.DateTime) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) is_private = db.Column(db.Boolean) parent_id = db.Column(UUID(), db.ForeignKey('paste.id')) parent = db.relationship('Paste', lazy=True, backref='children', uselist=False, remote_side=[id]) - def __init__(self, user, code, title, parent=None, is_private=False): + def __init__(self, user, code, title, relative_expiration_seconds, parent=None, is_private=False): self.user = user self.code = code self.title = title self.is_private = is_private - self.pub_date = datetime.utcnow() + u = datetime.utcnow() + try: + # this will fail on POSTed value of "never" for exp + b = timedelta(seconds=relative_expiration_seconds) + except: + # so timedelta() will return 0 seconds, which makes the exp_date = pub_date, which is treated + # by the cleanup logic and jinja2 templates as never-expires. + b = timedelta() + self.pub_date = u + self.exp_date = u + b self.parent = parent class User(db.Model): @@ -99,13 +117,62 @@ def new_paste(): title = "Untitled paste" if request.form['pastetitle'] and request.form['pastetitle'] != "Enter title here": title = request.form['pastetitle'] - paste = Paste(g.user, request.form['code'], title, parent=parent, is_private=is_private) + relative_expiration_seconds = 0 + exp = 0 # start with an empty value + if 'exp' in request.form and request.form['exp']: + exp_opt = request.form['exp'] + if exp_opt not in app.config['EXPIRATION_OPTIONS']: + try: + exp = timeparse(f"+ {app.config['EXPIRATION_OPTIONS'][0]}") + except: + exp = 60 * 60 # failsafe, 1 hour + print(f"WARNING: requested expiration \"{exp_opt}\" is not in the list of options {app.config['EXPIRATION_OPTIONS']}, so will use {exp}") + else: + try: + exp = timeparse(f"+ {exp_opt}") + except: + exp = 0 + paste = Paste( + g.user, + request.form['code'], + title, + exp, + parent=parent, + is_private=is_private + ) db.session.add(paste) db.session.commit() sign = get_signed(paste.id, salt=app.config['SALT']) \ if is_private else None return redirect(url_for('show_paste', paste_id=paste.id, s=sign)) - return render_template('new_paste.html', parent=parent) + try: + exp_opts = app.config['EXPIRATION_OPTIONS'] + except: + exp_opts = None + return render_template('new_paste.html', parent=parent, exp_opts = exp_opts) + +# This @timer is from the uwsgidecorators +try: + @timer(app.config['LOOP_DELAY']) + def cleanup_expired_pastes(num): + # num is useless. + """ + Every LOOP_DELAY seconds, find any entries that have expired and then delete them. + """ + all1 = Paste.query.all() + need_commit = False + for p in all1: + # the exp_date != pub_date is very important, because anything with "never" expires + # is stored in the db as exp_date = pub_date + if p.exp_date and p.exp_date != p.pub_date and p.exp_date <= datetime.utcnow(): + print(f"INFO: deleting paste \"{p.title}\" with expiration {p.exp_date}.") + Paste.query.filter(Paste.id == p.id).delete() + need_commit = True + # only run the commit once! + if need_commit: + db.session.commit() +except: + pass @app.route('//') @app.route('/') @@ -188,6 +255,8 @@ def get_all_pastes(): p2 = { "id": p1.id, "title": p1.title, + "pub_date": p1.pub_date, + "exp_date": p1.exp_date, "private": private, "user_id": p1.user_id, "is_private": p1.is_private, @@ -203,3 +272,7 @@ def get_all_pastes(): def admin(): all_pastes = get_all_pastes() return render_template('admin.html', pastes = all_pastes) + +if __name__ == "__main__": + manager.add_command('runserver', Server(host=app.config["APP_HOST"], port=app.config["APP_PORT"])) + app.run() diff --git a/stackbin.bin b/stackbin.bin new file mode 100755 index 0000000..d3b2da3 --- /dev/null +++ b/stackbin.bin @@ -0,0 +1,9 @@ +#!/bin/sh +# Reference: fuss.bin from fuss project +# Startdate: 2022-02-13 19:25 +# Goal: see if uwsgi reacts to the @timer directives in the pastebin.py file, because flask run doesn't. +thisscript="$( readlink -f "${0}" )" +COMMAND="" +grep -qiE 'ID=.*(rhel|centos|fedora)' /etc/os-release && COMMAND="${COMMAND} uwsgi" || \ + COMMAND="${COMMAND} uwsgi_python39" +${COMMAND} --ini "$( dirname "${thisscript}" )/$( basename "${thisscript}" | sed -r -e 's/\.bin$//;' ).wsgi.ini" diff --git a/stackbin.wsgi.ini b/stackbin.wsgi.ini new file mode 100644 index 0000000..cac4511 --- /dev/null +++ b/stackbin.wsgi.ini @@ -0,0 +1,14 @@ +[uwsgi] +plugins = logfile +http-socket = 0.0.0.0:5000 +wsgi-file = pastebin.py +callable = app +touch-reload = pastebin.py +touch-reload = config.cfg +touch-reload = stackbin.wsgi.ini +req-logger = file:log/req.log +# to get strftime format fields, you need double percent signs +logdate = "%%FT%%T" +logger = file:log/stackbin.log +# the init script uses a different pidfile owned by root. +pidfile = stackbin.pid diff --git a/static/style.css b/static/style.css index 4507323..874cadf 100644 --- a/static/style.css +++ b/static/style.css @@ -9,6 +9,7 @@ a:hover { color: #c00; } .nav li + li:before { content: " // "; } h2 { font-weight: normal; color: #222; margin-bottom: 10px; padding-left: 10px} .chk-private { float: right; font-size: 12px; margin-top: 12px} +.drop-expiration { float: left; font-size: 12px; margin-top: 12px} dl { overflow: auto; font-size: 14px; padding: 0 10px} dl dt { font-weight: bold; min-width: 70px; float: left; padding-right: 15px; clear: left; } diff --git a/templates/admin.html b/templates/admin.html index a815fb5..4991d7c 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -4,13 +4,15 @@

Administration for {{ appname }}

{% if pastes %} - +{##} {% for p in pastes %} - +{# #} {# magic string is from utf8icons.com #} + +
idprivatetitleuserparentchildrenActions
idprivatetitleuserpublishedexpiresparentchildrenActions
{{ p.id }}{{ p.id }}{% if p.private %}✓{% endif %}{{ p.title }} {% if p.user %}{{ p.user }}{% endif%}{{ p.pub_date.strftime('%FT%TZ') }}{% if p.exp_date != p.pub_date %}{{ p.exp_date.strftime('%FT%TZ') }}{% endif %} {% if p.parent[0] %}{{ p.parent[1] }}{% endif %} {% if p.children %}{% for c in p.children %}{% if not loop.first %},{% endif %} {{ c[1] }}{% endfor %}{% endif %} diff --git a/templates/new_paste.html b/templates/new_paste.html index 977ef22..b5a78aa 100644 --- a/templates/new_paste.html +++ b/templates/new_paste.html @@ -2,11 +2,14 @@ {% block title %}New Paste{% endblock %} {% block body %}
-

+

{%- if parent %} - Reply to {{ parent.title }} {%- endif %} - Private + + {% if exp_opts %}{% endif %}

diff --git a/templates/show_paste.html b/templates/show_paste.html index 03d57b1..e9ed02a 100644 --- a/templates/show_paste.html +++ b/templates/show_paste.html @@ -9,7 +9,9 @@

{{ paste.user.display_name }} {% endif %}
Date -
{{ paste.pub_date.strftime('%Y-%m-%dT%H:%M:%SZ') }} +
{{ paste.pub_date.strftime('%FT%TZ') }} + {% if paste.exp_date and paste.exp_date != paste.pub_date %}
Expires +
{{ paste.exp_date.strftime('%FT%TZ') }}{% endif %}
Actions
{% if not paste.is_private %} -- cgit