From 49b199c610bb32d0581d771d856b3e3332707f17 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Mon, 6 Apr 2020 23:05:31 +0200 Subject: Migrate form Flask-Script to the built-in integration of the click command line interface of Flask. --- README.md | 20 +++++++--- docker-compose.yml | 2 +- manager.py | 98 ---------------------------------------------- newspipe/bootstrap.py | 2 +- newspipe/commands.py | 85 ++++++++++++++++++++++++++++++++++++++++ newspipe/lib/misc_utils.py | 8 ++-- poetry.lock | 16 +------- pyproject.toml | 1 - runserver.py | 11 ++++++ wait-for-postgres.sh | 9 +++-- 10 files changed, 123 insertions(+), 129 deletions(-) delete mode 100755 manager.py create mode 100755 newspipe/commands.py diff --git a/README.md b/README.md index bdc92fd5..91cf7922 100644 --- a/README.md +++ b/README.md @@ -51,13 +51,21 @@ $ git clone https://git.sr.ht/~cedric/newspipe $ cd newspipe/ $ npm install $ poetry install -$ export Newspipe_CONFIG=sqlite.py $ poetry shell $ pybabel compile -d newspipe/translations -$ python manager.py db_create -$ python manager.py create_admin -$ python runserver.py - * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) +$ export NEWSPIPE_CONFIG=sqlite.py +$ export FLASK_APP=runserver.py +$ export FLASK_ENV=development +$ flask db_create +$ flask create_admin --nickname --password +$ flask run + * Serving Flask app "runserver" (lazy loading) + * Environment: development + * Debug mode: on + * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit) + * Restarting with stat + * Debugger is active! + * Debugger PIN: 221-873-938 ``` If you want to use PostgreSQL you can customize @@ -67,7 +75,7 @@ the provided example configuration file (``instance/config.py``): $ sudo apt-get install postgresql $ cp instance/config.py instance/postgresql.py $ vim instance/postgresql.py # customize it -$ export Newspipe_CONFIG=postgresql.py +$ export NEWSPIPE_CONFIG=postgresql.py ``` For production you can use [Gunicorn](https://gunicorn.org) or ``mod_wsgi``. diff --git a/docker-compose.yml b/docker-compose.yml index fd3bd0e0..efc52627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: dockerfile: ./Dockerfile tty: true environment: - - Newspipe_CONFIG=/newspipe/instance/config.py + - NEWSPIPE_CONFIG=/newspipe/instance/config.py ports: - "5000:5000" expose: diff --git a/manager.py b/manager.py deleted file mode 100755 index fcb5d0c5..00000000 --- a/manager.py +++ /dev/null @@ -1,98 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -import logging -import os -from datetime import datetime - -from flask_migrate import Migrate, MigrateCommand -from flask_script import Manager -from werkzeug.security import generate_password_hash - -import newspipe.models -from newspipe.bootstrap import application, db -from newspipe.controllers import UserController - -logger = logging.getLogger("manager") - -Migrate(application, db) - -manager = Manager(application) -manager.add_command("db", MigrateCommand) - - -@manager.command -def db_empty(): - "Will drop every datas stocked in db." - with application.app_context(): - newspipe.models.db_empty(db) - - -@manager.command -def db_create(): - "Will create the database from conf parameters." - admin = { - "is_admin": True, - "is_api": True, - "is_active": True, - "nickname": "admin", - "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")), - } - with application.app_context(): - try: - db.create_all() - UserController(ignore_context=True).create(**admin) - except Exception as e: - print(e) - - -@manager.command -def create_admin(nickname, password): - "Will create an admin user." - admin = { - "is_admin": True, - "is_api": True, - "is_active": True, - "nickname": nickname, - "pwdhash": generate_password_hash(password), - } - with application.app_context(): - UserController(ignore_context=True).create(**admin) - - -@manager.command -def fetch_asyncio(user_id=None, feed_id=None): - "Crawl the feeds with asyncio." - import asyncio - - with application.app_context(): - from newspipe.crawler import default_crawler - - filters = {} - filters["is_active"] = True - filters["automatic_crawling"] = True - if None is not user_id: - filters["id"] = user_id - users = UserController().read(**filters).all() - - try: - feed_id = int(feed_id) - except: - feed_id = None - - loop = asyncio.get_event_loop() - queue = asyncio.Queue(maxsize=3, loop=loop) - - producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) - consumer_coro = default_crawler.insert_articles(queue, 1) - - logger.info("Starting crawler.") - start = datetime.now() - loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) - end = datetime.now() - loop.close() - logger.info("Crawler finished in {} seconds.".format((end - start).seconds)) - - -if __name__ == "__main__": - manager.run() diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py index 70512493..e54aad7a 100644 --- a/newspipe/bootstrap.py +++ b/newspipe/bootstrap.py @@ -48,7 +48,7 @@ def set_logging( # Create Flask application application = Flask(__name__, instance_relative_config=True) -configuration = os.environ.get("Newspipe_CONFIG", False) +configuration = os.environ.get("NEWSPIPE_CONFIG", False) if configuration == "testing": application.debug = logging.DEBUG application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" diff --git a/newspipe/commands.py b/newspipe/commands.py new file mode 100755 index 00000000..62a355e4 --- /dev/null +++ b/newspipe/commands.py @@ -0,0 +1,85 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +import logging +import os +from datetime import datetime + +import click +from flask import Flask +from werkzeug.security import generate_password_hash + +import newspipe.models +from newspipe.bootstrap import application, db +from newspipe.controllers import UserController + +logger = logging.getLogger("commands") + + +@application.cli.command("db_empty") +def db_empty(): + "Will drop every datas stocked in db." + with application.app_context(): + newspipe.models.db_empty(db) + + +@application.cli.command("db_create") +def db_create(): + "Will create the database from conf parameters." + with application.app_context(): + try: + db.create_all() + except Exception as e: + print(e) + + +@application.cli.command("create_admin") +@click.option('--nickname', default='admin', help='Nickname') +@click.option('--password', default='password', help='Password') +def create_admin(nickname, password): + "Will create an admin user." + admin = { + "is_admin": True, + "is_api": True, + "is_active": True, + "nickname": nickname, + "pwdhash": generate_password_hash(password), + } + with application.app_context(): + UserController(ignore_context=True).create(**admin) + + +@application.cli.command("fetch_asyncio") +@click.option('--user-id', default=None, help='Id of the user') +@click.option('--feed-id', default=None, help='If of the feed') +def fetch_asyncio(user_id=None, feed_id=None): + "Crawl the feeds with asyncio." + import asyncio + + with application.app_context(): + from newspipe.crawler import default_crawler + + filters = {} + filters["is_active"] = True + filters["automatic_crawling"] = True + if None is not user_id: + filters["id"] = user_id + users = UserController().read(**filters).all() + + try: + feed_id = int(feed_id) + except: + feed_id = None + + loop = asyncio.get_event_loop() + queue = asyncio.Queue(maxsize=3, loop=loop) + + producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) + consumer_coro = default_crawler.insert_articles(queue, 1) + + logger.info("Starting crawler.") + start = datetime.now() + loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) + end = datetime.now() + loop.close() + logger.info("Crawler finished in {} seconds.".format((end - start).seconds)) diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py index bde65fd5..8eb03816 100755 --- a/newspipe/lib/misc_utils.py +++ b/newspipe/lib/misc_utils.py @@ -101,13 +101,13 @@ def fetch(id, feed_id=None): The default crawler ("asyncio") is launched with the manager. """ cmd = [ - sys.executable, - "manager.py", + "poetry", "run", + "flask", "fetch_asyncio", - "--user_id=" + str(id), + "--user-id", str(id), ] if feed_id: - cmd.append("--feed_id=" + str(feed_id)) + cmd.extend(["--feed-id", str(feed_id)]) return subprocess.Popen(cmd, stdout=subprocess.PIPE) diff --git a/poetry.lock b/poetry.lock index bb822dd5..4ff82f23 100644 --- a/poetry.lock +++ b/poetry.lock @@ -245,17 +245,6 @@ six = ">=1.3.0" [package.extras] docs = ["sphinx"] -[[package]] -category = "main" -description = "Scripting support for Flask" -name = "flask-script" -optional = false -python-versions = "*" -version = "2.0.6" - -[package.dependencies] -Flask = "*" - [[package]] category = "main" description = "Adds SQLAlchemy support to your Flask application." @@ -583,7 +572,7 @@ idna = ">=2.0" multidict = ">=4.0" [metadata] -content-hash = "c8407863562e0f8573d3f8b8a7b1ab4b09ea3a40271ae077af278176246e934b" +content-hash = "f5e5860ca483252734221f791fc859169287b8c691ab6ee15816f79794faabcd" python-versions = "^3.8" [metadata.files] @@ -679,9 +668,6 @@ flask-restful = [ {file = "Flask-RESTful-0.3.8.tar.gz", hash = "sha256:5ea9a5991abf2cb69b4aac19793faac6c032300505b325687d7c305ffaa76915"}, {file = "Flask_RESTful-0.3.8-py2.py3-none-any.whl", hash = "sha256:d891118b951921f1cec80cabb4db98ea6058a35e6404788f9e70d5b243813ec2"}, ] -flask-script = [ - {file = "Flask-Script-2.0.6.tar.gz", hash = "sha256:6425963d91054cfcc185807141c7314a9c5ad46325911bd24dcb489bd0161c65"}, -] flask-sqlalchemy = [ {file = "Flask-SQLAlchemy-2.4.1.tar.gz", hash = "sha256:6974785d913666587949f7c2946f7001e4fa2cb2d19f4e69ead02e4b8f50b33d"}, {file = "Flask_SQLAlchemy-2.4.1-py2.py3-none-any.whl", hash = "sha256:0078d8663330dc05a74bc72b3b6ddc441b9a744e2f56fe60af1a5bfc81334327"}, diff --git a/pyproject.toml b/pyproject.toml index f6af71a3..70fe0840 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ Flask-RESTful = "^0.3.8" Flask-paginate = "^0.5.5" Flask-Babel = "^1.0.0" Flask-Migrate = "^2.5.2" -Flask-Script = "^2.0.6" WTForms = "^2.2.1" python-dateutil = "^2.8.1" psycopg2-binary = "^2.8.4" diff --git a/runserver.py b/runserver.py index b3f02823..2fe093f0 100755 --- a/runserver.py +++ b/runserver.py @@ -22,6 +22,15 @@ from flask import g from flask_restful import Api from newspipe.bootstrap import application +from newspipe import commands + + +def register_commands(app): + """Register Click commands.""" + app.cli.add_command(commands.db_empty) + app.cli.add_command(commands.db_create) + app.cli.add_command(commands.fetch_asyncio) + app.cli.add_command(commands.create_admin) with application.app_context(): @@ -42,6 +51,8 @@ with application.app_context(): application.register_blueprint(views.bookmarks_bp) application.register_blueprint(views.bookmark_bp) + register_commands(application) + if __name__ == "__main__": application.run( diff --git a/wait-for-postgres.sh b/wait-for-postgres.sh index 3f4e7067..be5e8afe 100755 --- a/wait-for-postgres.sh +++ b/wait-for-postgres.sh @@ -13,7 +13,10 @@ do done >&2 echo "Postgres is up - executing command" -export Newspipe_CONFIG=/newspipe/instance/config.py -poetry run ./manager.py db_create >/dev/null +export NEWSPIPE_CONFIG=/newspipe/instance/config.py +export FLASK_APP=runserver.py +export FLASK_ENV=development +poetry run flask db_create >/dev/null +poetry run flask create_admin --nickname admin --password password >/dev/null poetry run pybabel compile -d newspipe/translations -poetry run ./runserver.py +poetry run flask run -- cgit