From 3ab6290d4994b33cdbf831523938cdb18a13bf49 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Mon, 9 Mar 2020 23:16:05 +0100 Subject: Refactoring the backend. --- .gitignore | 6 +- instance/production.py | 56 ++++++ manager.py | 94 ++++++++++ newspipe/bootstrap.py | 49 +++--- newspipe/conf.py | 2 +- newspipe/conf/conf.cfg-sample | 31 ---- newspipe/controllers/__init__.py | 18 ++ newspipe/controllers/abstract.py | 177 +++++++++++++++++++ newspipe/controllers/article.py | 107 ++++++++++++ newspipe/controllers/bookmark.py | 35 ++++ newspipe/controllers/category.py | 13 ++ newspipe/controllers/feed.py | 105 +++++++++++ newspipe/controllers/icon.py | 27 +++ newspipe/controllers/tag.py | 22 +++ newspipe/controllers/user.py | 28 +++ newspipe/crawler/default_crawler.py | 24 +-- newspipe/lib/article_utils.py | 10 +- newspipe/lib/data.py | 8 +- newspipe/lib/feed_utils.py | 6 +- newspipe/lib/misc_utils.py | 10 +- newspipe/lib/utils.py | 10 +- newspipe/manager.py | 94 ---------- newspipe/models/__init__.py | 98 +++++++++++ newspipe/models/article.py | 101 +++++++++++ newspipe/models/bookmark.py | 72 ++++++++ newspipe/models/category.py | 28 +++ newspipe/models/feed.py | 105 +++++++++++ newspipe/models/icon.py | 10 ++ newspipe/models/right_mixin.py | 64 +++++++ newspipe/models/role.py | 40 +++++ newspipe/models/tag.py | 44 +++++ newspipe/models/user.py | 114 ++++++++++++ newspipe/notifications/emails.py | 16 +- newspipe/notifications/notifications.py | 15 +- newspipe/runserver.py | 66 ------- newspipe/static/css/custom.css | 80 +++++++++ newspipe/static/img/favicon.ico | Bin 0 -> 1150 bytes newspipe/static/img/newspipe.png | Bin 0 -> 1547 bytes newspipe/static/img/newspipe.svg | 84 +++++++++ newspipe/static/img/pinboard.png | Bin 0 -> 597 bytes newspipe/static/img/reddit.png | Bin 0 -> 525 bytes newspipe/static/img/twitter.png | Bin 0 -> 640 bytes newspipe/static/js/articles.js | 191 +++++++++++++++++++++ newspipe/static/js/feed.js | 22 +++ newspipe/templates/about.html | 26 +++ newspipe/templates/about_more.html | 11 ++ newspipe/templates/admin/create_user.html | 26 +++ newspipe/templates/admin/dashboard.html | 68 ++++++++ newspipe/templates/article.html | 35 ++++ newspipe/templates/article_pub.html | 24 +++ newspipe/templates/bookmarks.html | 81 +++++++++ newspipe/templates/categories.html | 36 ++++ newspipe/templates/duplicates.html | 30 ++++ newspipe/templates/edit_bookmark.html | 84 +++++++++ newspipe/templates/edit_category.html | 23 +++ newspipe/templates/edit_feed.html | 98 +++++++++++ newspipe/templates/emails/account_activation.txt | 9 + newspipe/templates/emails/new_password.txt | 8 + newspipe/templates/errors/404.html | 12 ++ newspipe/templates/errors/500.html | 12 ++ newspipe/templates/feed.html | 76 ++++++++ newspipe/templates/feed_list.html | 55 ++++++ newspipe/templates/feed_list_per_categories.html | 52 ++++++ newspipe/templates/feed_list_simple.html | 35 ++++ newspipe/templates/feeds.html | 7 + newspipe/templates/history.html | 26 +++ newspipe/templates/home.html | 135 +++++++++++++++ newspipe/templates/inactives.html | 26 +++ newspipe/templates/layout.html | 140 +++++++++++++++ newspipe/templates/login.html | 32 ++++ newspipe/templates/management.html | 99 +++++++++++ newspipe/templates/opml.xml | 13 ++ newspipe/templates/popular.html | 27 +++ newspipe/templates/profile.html | 68 ++++++++ newspipe/templates/profile_public.html | 45 +++++ newspipe/templates/signup.html | 24 +++ newspipe/templates/user_stream.html | 70 ++++++++ newspipe/web/controllers/__init__.py | 18 -- newspipe/web/controllers/abstract.py | 177 ------------------- newspipe/web/controllers/article.py | 107 ------------ newspipe/web/controllers/bookmark.py | 35 ---- newspipe/web/controllers/category.py | 13 -- newspipe/web/controllers/feed.py | 105 ----------- newspipe/web/controllers/icon.py | 27 --- newspipe/web/controllers/tag.py | 22 --- newspipe/web/controllers/user.py | 28 --- newspipe/web/forms.py | 6 +- newspipe/web/lib/user_utils.py | 6 +- newspipe/web/lib/view_utils.py | 2 +- newspipe/web/models/__init__.py | 98 ----------- newspipe/web/models/article.py | 101 ----------- newspipe/web/models/bookmark.py | 72 -------- newspipe/web/models/category.py | 28 --- newspipe/web/models/feed.py | 105 ----------- newspipe/web/models/icon.py | 10 -- newspipe/web/models/right_mixin.py | 64 ------- newspipe/web/models/role.py | 40 ----- newspipe/web/models/tag.py | 44 ----- newspipe/web/models/user.py | 114 ------------ newspipe/web/static/css/custom.css | 80 --------- newspipe/web/static/img/favicon.ico | Bin 1150 -> 0 bytes newspipe/web/static/img/newspipe.png | Bin 1547 -> 0 bytes newspipe/web/static/img/newspipe.svg | 84 --------- newspipe/web/static/img/pinboard.png | Bin 597 -> 0 bytes newspipe/web/static/img/reddit.png | Bin 525 -> 0 bytes newspipe/web/static/img/twitter.png | Bin 640 -> 0 bytes newspipe/web/static/js/articles.js | 191 --------------------- newspipe/web/static/js/feed.js | 22 --- newspipe/web/templates/about.html | 26 --- newspipe/web/templates/about_more.html | 11 -- newspipe/web/templates/admin/create_user.html | 26 --- newspipe/web/templates/admin/dashboard.html | 68 -------- newspipe/web/templates/article.html | 35 ---- newspipe/web/templates/article_pub.html | 24 --- newspipe/web/templates/bookmarks.html | 81 --------- newspipe/web/templates/categories.html | 36 ---- newspipe/web/templates/duplicates.html | 30 ---- newspipe/web/templates/edit_bookmark.html | 84 --------- newspipe/web/templates/edit_category.html | 23 --- newspipe/web/templates/edit_feed.html | 98 ----------- .../web/templates/emails/account_activation.txt | 9 - newspipe/web/templates/emails/new_password.txt | 8 - newspipe/web/templates/errors/404.html | 12 -- newspipe/web/templates/errors/500.html | 12 -- newspipe/web/templates/feed.html | 76 -------- newspipe/web/templates/feed_list.html | 55 ------ .../web/templates/feed_list_per_categories.html | 52 ------ newspipe/web/templates/feed_list_simple.html | 35 ---- newspipe/web/templates/feeds.html | 7 - newspipe/web/templates/history.html | 26 --- newspipe/web/templates/home.html | 135 --------------- newspipe/web/templates/inactives.html | 26 --- newspipe/web/templates/layout.html | 140 --------------- newspipe/web/templates/login.html | 32 ---- newspipe/web/templates/management.html | 99 ----------- newspipe/web/templates/opml.xml | 13 -- newspipe/web/templates/popular.html | 27 --- newspipe/web/templates/profile.html | 68 -------- newspipe/web/templates/profile_public.html | 45 ----- newspipe/web/templates/signup.html | 24 --- newspipe/web/templates/user_stream.html | 70 -------- newspipe/web/views/__init__.py | 46 +---- newspipe/web/views/admin.py | 8 +- newspipe/web/views/api/v2/__init__.py | 2 +- newspipe/web/views/api/v2/article.py | 10 +- newspipe/web/views/api/v2/category.py | 8 +- newspipe/web/views/api/v2/common.py | 4 +- newspipe/web/views/api/v2/feed.py | 10 +- newspipe/web/views/article.py | 10 +- newspipe/web/views/bookmark.py | 13 +- newspipe/web/views/category.py | 8 +- newspipe/web/views/common.py | 4 +- newspipe/web/views/feed.py | 18 +- newspipe/web/views/home.py | 14 +- newspipe/web/views/icon.py | 5 +- newspipe/web/views/session_mgmt.py | 12 +- newspipe/web/views/user.py | 16 +- newspipe/web/views/views.py | 17 +- package-lock.json | 6 +- package.json | 2 +- runserver.py | 68 ++++++++ 161 files changed, 3491 insertions(+), 3497 deletions(-) create mode 100644 instance/production.py create mode 100755 manager.py delete mode 100644 newspipe/conf/conf.cfg-sample create mode 100644 newspipe/controllers/__init__.py create mode 100644 newspipe/controllers/abstract.py create mode 100644 newspipe/controllers/article.py create mode 100644 newspipe/controllers/bookmark.py create mode 100644 newspipe/controllers/category.py create mode 100644 newspipe/controllers/feed.py create mode 100644 newspipe/controllers/icon.py create mode 100644 newspipe/controllers/tag.py create mode 100644 newspipe/controllers/user.py delete mode 100755 newspipe/manager.py create mode 100644 newspipe/models/__init__.py create mode 100644 newspipe/models/article.py create mode 100644 newspipe/models/bookmark.py create mode 100644 newspipe/models/category.py create mode 100644 newspipe/models/feed.py create mode 100644 newspipe/models/icon.py create mode 100644 newspipe/models/right_mixin.py create mode 100644 newspipe/models/role.py create mode 100644 newspipe/models/tag.py create mode 100644 newspipe/models/user.py delete mode 100755 newspipe/runserver.py create mode 100644 newspipe/static/css/custom.css create mode 100644 newspipe/static/img/favicon.ico create mode 100644 newspipe/static/img/newspipe.png create mode 100644 newspipe/static/img/newspipe.svg create mode 100644 newspipe/static/img/pinboard.png create mode 100755 newspipe/static/img/reddit.png create mode 100644 newspipe/static/img/twitter.png create mode 100644 newspipe/static/js/articles.js create mode 100644 newspipe/static/js/feed.js create mode 100644 newspipe/templates/about.html create mode 100644 newspipe/templates/about_more.html create mode 100644 newspipe/templates/admin/create_user.html create mode 100644 newspipe/templates/admin/dashboard.html create mode 100644 newspipe/templates/article.html create mode 100644 newspipe/templates/article_pub.html create mode 100644 newspipe/templates/bookmarks.html create mode 100644 newspipe/templates/categories.html create mode 100644 newspipe/templates/duplicates.html create mode 100644 newspipe/templates/edit_bookmark.html create mode 100644 newspipe/templates/edit_category.html create mode 100644 newspipe/templates/edit_feed.html create mode 100644 newspipe/templates/emails/account_activation.txt create mode 100644 newspipe/templates/emails/new_password.txt create mode 100644 newspipe/templates/errors/404.html create mode 100644 newspipe/templates/errors/500.html create mode 100644 newspipe/templates/feed.html create mode 100644 newspipe/templates/feed_list.html create mode 100644 newspipe/templates/feed_list_per_categories.html create mode 100644 newspipe/templates/feed_list_simple.html create mode 100644 newspipe/templates/feeds.html create mode 100644 newspipe/templates/history.html create mode 100644 newspipe/templates/home.html create mode 100644 newspipe/templates/inactives.html create mode 100644 newspipe/templates/layout.html create mode 100644 newspipe/templates/login.html create mode 100644 newspipe/templates/management.html create mode 100644 newspipe/templates/opml.xml create mode 100644 newspipe/templates/popular.html create mode 100644 newspipe/templates/profile.html create mode 100644 newspipe/templates/profile_public.html create mode 100644 newspipe/templates/signup.html create mode 100644 newspipe/templates/user_stream.html delete mode 100644 newspipe/web/controllers/__init__.py delete mode 100644 newspipe/web/controllers/abstract.py delete mode 100644 newspipe/web/controllers/article.py delete mode 100644 newspipe/web/controllers/bookmark.py delete mode 100644 newspipe/web/controllers/category.py delete mode 100644 newspipe/web/controllers/feed.py delete mode 100644 newspipe/web/controllers/icon.py delete mode 100644 newspipe/web/controllers/tag.py delete mode 100644 newspipe/web/controllers/user.py delete mode 100644 newspipe/web/models/__init__.py delete mode 100644 newspipe/web/models/article.py delete mode 100644 newspipe/web/models/bookmark.py delete mode 100644 newspipe/web/models/category.py delete mode 100644 newspipe/web/models/feed.py delete mode 100644 newspipe/web/models/icon.py delete mode 100644 newspipe/web/models/right_mixin.py delete mode 100644 newspipe/web/models/role.py delete mode 100644 newspipe/web/models/tag.py delete mode 100644 newspipe/web/models/user.py delete mode 100644 newspipe/web/static/css/custom.css delete mode 100644 newspipe/web/static/img/favicon.ico delete mode 100644 newspipe/web/static/img/newspipe.png delete mode 100644 newspipe/web/static/img/newspipe.svg delete mode 100644 newspipe/web/static/img/pinboard.png delete mode 100755 newspipe/web/static/img/reddit.png delete mode 100644 newspipe/web/static/img/twitter.png delete mode 100644 newspipe/web/static/js/articles.js delete mode 100644 newspipe/web/static/js/feed.js delete mode 100644 newspipe/web/templates/about.html delete mode 100644 newspipe/web/templates/about_more.html delete mode 100644 newspipe/web/templates/admin/create_user.html delete mode 100644 newspipe/web/templates/admin/dashboard.html delete mode 100644 newspipe/web/templates/article.html delete mode 100644 newspipe/web/templates/article_pub.html delete mode 100644 newspipe/web/templates/bookmarks.html delete mode 100644 newspipe/web/templates/categories.html delete mode 100644 newspipe/web/templates/duplicates.html delete mode 100644 newspipe/web/templates/edit_bookmark.html delete mode 100644 newspipe/web/templates/edit_category.html delete mode 100644 newspipe/web/templates/edit_feed.html delete mode 100644 newspipe/web/templates/emails/account_activation.txt delete mode 100644 newspipe/web/templates/emails/new_password.txt delete mode 100644 newspipe/web/templates/errors/404.html delete mode 100644 newspipe/web/templates/errors/500.html delete mode 100644 newspipe/web/templates/feed.html delete mode 100644 newspipe/web/templates/feed_list.html delete mode 100644 newspipe/web/templates/feed_list_per_categories.html delete mode 100644 newspipe/web/templates/feed_list_simple.html delete mode 100644 newspipe/web/templates/feeds.html delete mode 100644 newspipe/web/templates/history.html delete mode 100644 newspipe/web/templates/home.html delete mode 100644 newspipe/web/templates/inactives.html delete mode 100644 newspipe/web/templates/layout.html delete mode 100644 newspipe/web/templates/login.html delete mode 100644 newspipe/web/templates/management.html delete mode 100644 newspipe/web/templates/opml.xml delete mode 100644 newspipe/web/templates/popular.html delete mode 100644 newspipe/web/templates/profile.html delete mode 100644 newspipe/web/templates/profile_public.html delete mode 100644 newspipe/web/templates/signup.html delete mode 100644 newspipe/web/templates/user_stream.html create mode 100755 runserver.py diff --git a/.gitignore b/.gitignore index fc5c2c51..27f1cf1f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,10 +24,10 @@ build newspipe.egg-info/ -newspipe/conf/conf.cfg +instance/development.py .coverage # js and node files node_modules -newspipe/web/static/npm_components -newspipe/web/static/js/bundle.min.js +newspipe/static/npm_components +newspipe/static/js/bundle.min.js diff --git a/instance/production.py b/instance/production.py new file mode 100644 index 00000000..af2f1c1e --- /dev/null +++ b/instance/production.py @@ -0,0 +1,56 @@ +# [webserver] +HOST = '127.0.0.1' +PORT = 5000 +SECRET_KEY = 'a secret only you know' +DEBUG = False +TESTING = False +API_ROOT = "/api/v2.0" + +SECRET_KEY = 'LCx3BchmHRxFzkEv4BqQJyeXRLXenf' +SECURITY_PASSWORD_SALT = 'L8gTsyrpRQEF8jNWQPyvRfv7U5kJkD' + + +# [misc] +ADMIN_EMAIL = 'admin@admin.localhost' +SECURITY_PASSWORD_SALT = 'a secret to confirm user account' +TOKEN_VALIDITY_PERIOD = 3600 +LOG_PATH = './var/newspipe.log' +NB_WORKER = 5 +DEBUG = False +TESTING = False +LOG_LEVEL = 'info' +SELF_REGISTRATION = True + + +# [database] +DB_CONFIG_DICT = { + 'user': 'user', + 'password': 'password', + 'host': 'localhost', + 'port': 5432 +} +DATABASE_NAME = 'newspipe' +SQLALCHEMY_DATABASE_URI = 'postgres://{user}:{password}@{host}:{port}/{name}'.format(name=DATABASE_NAME, **DB_CONFIG_DICT) +SQLALCHEMY_TRACK_MODIFICATIONS = False + + +# [crawler] +CRAWLING_METHOD = 'default' +DEFAULT_MAX_ERROR = 3 +HTTP_PROXY = '' +USER_AGENT = 'JARR (https://github.com/JARR/JARR)' +RESOLVE_ARTICLE_URL = False +TIMEOUT = 30 +RESOLV = False +FEED_REFRESH_INTERVAL = 0 + + +# [notification] +MAIL_SERVER = 'localhost' +MAIL_PORT = 25 +MAIL_USE_TLS = False +MAIL_USE_SSL = False +MAIL_DEBUG = DEBUG +MAIL_USERNAME = None +MAIL_PASSWORD = None +MAIL_DEFAULT_SENDER = ADMIN_EMAIL diff --git a/manager.py b/manager.py new file mode 100755 index 00000000..bf935632 --- /dev/null +++ b/manager.py @@ -0,0 +1,94 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import logging +from datetime import datetime +from werkzeug.security import generate_password_hash +from flask_script import Manager +from flask_migrate import Migrate, MigrateCommand + +import newspipe.models +from newspipe.controllers import UserController +from newspipe.bootstrap import application, db, set_logging + +logger = logging.getLogger("manager") + +Migrate(application, db) + +manager = Manager(application) +manager.add_command("db", MigrateCommand) + + +@manager.command +def db_empty(): + "Will drop every datas stocked in db." + with application.app_context(): + web.models.db_empty(db) + + +@manager.command +def db_create(): + "Will create the database from conf parameters." + admin = { + "is_admin": True, + "is_api": True, + "is_active": True, + "nickname": "admin", + "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")), + } + with application.app_context(): + db.create_all() + UserController(ignore_context=True).create(**admin) + + +@manager.command +def create_admin(nickname, password): + "Will create an admin user." + admin = { + "is_admin": True, + "is_api": True, + "is_active": True, + "nickname": nickname, + "pwdhash": generate_password_hash(password), + } + with application.app_context(): + UserController(ignore_context=True).create(**admin) + + +@manager.command +def fetch_asyncio(user_id=None, feed_id=None): + "Crawl the feeds with asyncio." + import asyncio + + with application.app_context(): + from newspipe.crawler import default_crawler + + filters = {} + filters["is_active"] = True + filters["automatic_crawling"] = True + if None is not user_id: + filters["id"] = user_id + users = UserController().read(**filters).all() + + try: + feed_id = int(feed_id) + except: + feed_id = None + + loop = asyncio.get_event_loop() + queue = asyncio.Queue(maxsize=3, loop=loop) + + producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) + consumer_coro = default_crawler.insert_articles(queue, 1) + + logger.info("Starting crawler.") + start = datetime.now() + loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) + end = datetime.now() + loop.close() + logger.info("Crawler finished in {} seconds.".format((end - start).seconds)) + + +if __name__ == "__main__": + manager.run() diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py index cbdbc820..1fd1ab04 100644 --- a/newspipe/bootstrap.py +++ b/newspipe/bootstrap.py @@ -4,9 +4,9 @@ # required imports and code execution for basic functionning import os -import conf import logging from urllib.parse import urlsplit +from flask_babel import Babel, format_datetime def set_logging( @@ -47,41 +47,36 @@ from flask import Flask from flask_sqlalchemy import SQLAlchemy # Create Flask application -application = Flask("web") +application = Flask(__name__, instance_relative_config=True) if os.environ.get("Newspipe_TESTING", False) == "true": application.debug = logging.DEBUG application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" application.config["TESTING"] = True else: - application.debug = conf.LOG_LEVEL <= logging.DEBUG - application.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - application.config["SQLALCHEMY_DATABASE_URI"] = conf.SQLALCHEMY_DATABASE_URI - if "postgres" in conf.SQLALCHEMY_DATABASE_URI: - application.config["SQLALCHEMY_POOL_SIZE"] = 15 - application.config["SQLALCHEMY_MAX_OVERFLOW"] = 0 + try: + application.config.from_pyfile("development.py", silent=False) + except Exception: + application.config.from_pyfile("production.py", silent=False) -scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL) -application.config["SERVER_NAME"] = domain -application.config["PREFERRED_URL_SCHEME"] = scheme +# scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL) +# application.config["SERVER_NAME"] = domain +# application.config["PREFERRED_URL_SCHEME"] = scheme -set_logging(conf.LOG_PATH, log_level=conf.LOG_LEVEL) - -# Create secrey key so we can use sessions -application.config["SECRET_KEY"] = getattr(conf, "WEBSERVER_SECRET", None) -if not application.config["SECRET_KEY"]: - application.config["SECRET_KEY"] = os.urandom(12) - -application.config["SECURITY_PASSWORD_SALT"] = getattr( - conf, "SECURITY_PASSWORD_SALT", None -) -if not application.config["SECURITY_PASSWORD_SALT"]: - application.config["SECURITY_PASSWORD_SALT"] = os.urandom(12) +set_logging(application.config['LOG_PATH']) db = SQLAlchemy(application) -def populate_g(): - from flask import g +babel = Babel(application) + - g.db = db - g.app = application +@babel.localeselector +def get_locale(): + # if a user is logged in, use the locale from the user settings + # user = getattr(g, 'user', None) + # if user is not None: + # return user.locale + # otherwise try to guess the language from the user accept + # header the browser transmits. We support de/fr/en in this + # example. The best match wins. + return request.accept_languages.best_match(["fr", "en"]) diff --git a/newspipe/conf.py b/newspipe/conf.py index bb51e563..09a38be5 100644 --- a/newspipe/conf.py +++ b/newspipe/conf.py @@ -10,7 +10,7 @@ import logging BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PATH = os.path.abspath(".") -API_ROOT = "/api/v2.0" + # available languages LANGUAGES = {"en": "English", "fr": "French"} diff --git a/newspipe/conf/conf.cfg-sample b/newspipe/conf/conf.cfg-sample deleted file mode 100644 index 41035fc7..00000000 --- a/newspipe/conf/conf.cfg-sample +++ /dev/null @@ -1,31 +0,0 @@ -[webserver] -host = 0.0.0.0 -port = 5000 -secret_key = a secret only you know -debug = true -[cdn] -cdn_address = https://cdn.cedricbonhomme.org/ -[misc] -platform_url = http://127.0.0.1:5000/ -admin_email = -security_password_salt = a secret to confirm user account -token_validity_period = 3600 -log_path = ./var/log/newspipe.log -log_level = info -[database] -database_url = sqlite:///newspipe.db -[crawler] -crawling_method = default -default_max_error = 6 -user_agent = Newspipe (https://git.sr.ht/~cedric/newspipe) -timeout = 30 -resolv = false -feed_refresh_interval = 120 -[notification] -notification_email = Newspipe@no-reply.com -host = smtp.googlemail.com -port = 465 -tls = false -ssl = true -username = your-gmail-username -password = your-gmail-password diff --git a/newspipe/controllers/__init__.py b/newspipe/controllers/__init__.py new file mode 100644 index 00000000..9b193cc5 --- /dev/null +++ b/newspipe/controllers/__init__.py @@ -0,0 +1,18 @@ +from .feed import FeedController +from .category import CategoryController +from .article import ArticleController +from .user import UserController +from .icon import IconController +from .bookmark import BookmarkController +from .tag import BookmarkTagController + + +__all__ = [ + "FeedController", + "CategoryController", + "ArticleController", + "UserController", + "IconController", + "BookmarkController", + "BookmarkTagController", +] diff --git a/newspipe/controllers/abstract.py b/newspipe/controllers/abstract.py new file mode 100644 index 00000000..a699afd0 --- /dev/null +++ b/newspipe/controllers/abstract.py @@ -0,0 +1,177 @@ +import logging +import dateutil.parser +from newspipe.bootstrap import db +from datetime import datetime +from collections import defaultdict +from sqlalchemy import or_, func +from werkzeug.exceptions import Forbidden, NotFound + +logger = logging.getLogger(__name__) + + +class AbstractController: + _db_cls = None # reference to the database class + _user_id_key = "user_id" + + def __init__(self, user_id=None, ignore_context=False): + """User id is a right management mechanism that should be used to + filter objects in database on their denormalized "user_id" field + (or "id" field for users). + Should no user_id be provided, the Controller won't apply any filter + allowing for a kind of "super user" mode. + """ + try: + self.user_id = int(user_id) + except TypeError: + self.user_id = user_id + + def _to_filters(self, **filters): + """ + Will translate filters to sqlalchemy filter. + This method will also apply user_id restriction if available. + + each parameters of the function is treated as an equality unless the + name of the parameter ends with either "__gt", "__lt", "__ge", "__le", + "__ne", "__in" ir "__like". + """ + db_filters = set() + for key, value in filters.items(): + if key == "__or__": + db_filters.add(or_(*self._to_filters(**value))) + elif key.endswith("__gt"): + db_filters.add(getattr(self._db_cls, key[:-4]) > value) + elif key.endswith("__lt"): + db_filters.add(getattr(self._db_cls, key[:-4]) < value) + elif key.endswith("__ge"): + db_filters.add(getattr(self._db_cls, key[:-4]) >= value) + elif key.endswith("__le"): + db_filters.add(getattr(self._db_cls, key[:-4]) <= value) + elif key.endswith("__ne"): + db_filters.add(getattr(self._db_cls, key[:-4]) != value) + elif key.endswith("__in"): + db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) + elif key.endswith("__contains"): + db_filters.add(getattr(self._db_cls, key[:-10]).contains(value)) + elif key.endswith("__like"): + db_filters.add(getattr(self._db_cls, key[:-6]).like(value)) + elif key.endswith("__ilike"): + db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value)) + else: + db_filters.add(getattr(self._db_cls, key) == value) + return db_filters + + def _get(self, **filters): + """ Will add the current user id if that one is not none (in which case + the decision has been made in the code that the query shouldn't be user + dependent) and the user is not an admin and the filters doesn't already + contains a filter for that user. + """ + if ( + self._user_id_key is not None + and self.user_id + and filters.get(self._user_id_key) != self.user_id + ): + filters[self._user_id_key] = self.user_id + return self._db_cls.query.filter(*self._to_filters(**filters)) + + def get(self, **filters): + """Will return one single objects corresponding to filters""" + obj = self._get(**filters).first() + + if obj and not self._has_right_on(obj): + raise Forbidden( + { + "message": "No authorized to access %r (%r)" + % (self._db_cls.__class__.__name__, filters) + } + ) + if not obj: + raise NotFound( + {"message": "No %r (%r)" % (self._db_cls.__class__.__name__, filters)} + ) + return obj + + def create(self, **attrs): + assert attrs, "attributes to update must not be empty" + if self._user_id_key is not None and self._user_id_key not in attrs: + attrs[self._user_id_key] = self.user_id + assert ( + self._user_id_key is None + or self._user_id_key in attrs + or self.user_id is None + ), "You must provide user_id one way or another" + + obj = self._db_cls(**attrs) + db.session.add(obj) + db.session.flush() + db.session.commit() + return obj + + def read(self, **filters): + return self._get(**filters) + + def update(self, filters, attrs, return_objs=False, commit=True): + assert attrs, "attributes to update must not be empty" + result = self._get(**filters).update(attrs, synchronize_session=False) + if commit: + db.session.flush() + db.session.commit() + if return_objs: + return self._get(**filters) + return result + + def delete(self, obj_id): + obj = self.get(id=obj_id) + db.session.delete(obj) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + return obj + + def _has_right_on(self, obj): + # user_id == None is like being admin + if self._user_id_key is None: + return True + return ( + self.user_id is None + or getattr(obj, self._user_id_key, None) == self.user_id + ) + + def _count_by(self, elem_to_group_by, filters): + if self.user_id: + filters["user_id"] = self.user_id + return dict( + db.session.query(elem_to_group_by, func.count("id")) + .filter(*self._to_filters(**filters)) + .group_by(elem_to_group_by) + .all() + ) + + @classmethod + def _get_attrs_desc(cls, role, right=None): + result = defaultdict(dict) + if role == "admin": + columns = cls._db_cls.__table__.columns.keys() + else: + assert role in {"base", "api"}, "unknown role %r" % role + assert right in {"read", "write"}, ( + "right must be 'read' or 'write' with role %r" % role + ) + columns = getattr(cls._db_cls, "fields_%s_%s" % (role, right))() + for column in columns: + result[column] = {} + db_col = getattr(cls._db_cls, column).property.columns[0] + try: + result[column]["type"] = db_col.type.python_type + except NotImplementedError: + if db_col.default: + result[column]["type"] = db_col.default.arg.__class__ + if column not in result: + continue + if issubclass(result[column]["type"], datetime): + result[column]["default"] = datetime.utcnow() + result[column]["type"] = lambda x: dateutil.parser.parse(x) + elif db_col.default: + result[column]["default"] = db_col.default.arg + return result diff --git a/newspipe/controllers/article.py b/newspipe/controllers/article.py new file mode 100644 index 00000000..d4adf99d --- /dev/null +++ b/newspipe/controllers/article.py @@ -0,0 +1,107 @@ +import re +import logging +import sqlalchemy +from sqlalchemy import func +from collections import Counter + +from newspipe.bootstrap import db +from .abstract import AbstractController +from newspipe.lib.article_utils import process_filters +from newspipe.controllers import CategoryController, FeedController +from newspipe.models import Article + +logger = logging.getLogger(__name__) + + +class ArticleController(AbstractController): + _db_cls = Article + + def challenge(self, ids): + """Will return each id that wasn't found in the database.""" + for id_ in ids: + if self.read(**id_).first(): + continue + yield id_ + + def count_by_category(self, **filters): + return self._count_by(Article.category_id, filters) + + def count_by_feed(self, **filters): + return self._count_by(Article.feed_id, filters) + + def count_by_user_id(self, **filters): + return dict( + db.session.query(Article.user_id, func.count(Article.id)) + .filter(*self._to_filters(**filters)) + .group_by(Article.user_id) + .all() + ) + + def create(self, **attrs): + # handling special denorm for article rights + assert "feed_id" in attrs, "must provide feed_id when creating article" + feed = FeedController(attrs.get("user_id", self.user_id)).get( + id=attrs["feed_id"] + ) + if "user_id" in attrs: + assert feed.user_id == attrs["user_id"] or self.user_id is None, ( + "no right on feed %r" % feed.id + ) + attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id + + skipped, read, liked = process_filters(feed.filters, attrs) + if skipped: + return None + article = super().create(**attrs) + return article + + def update(self, filters, attrs): + user_id = attrs.get("user_id", self.user_id) + if "feed_id" in attrs: + feed = FeedController().get(id=attrs["feed_id"]) + assert feed.user_id == user_id, "no right on feed %r" % feed.id + attrs["category_id"] = feed.category_id + if attrs.get("category_id"): + cat = CategoryController().get(id=attrs["category_id"]) + assert self.user_id is None or cat.user_id == user_id, ( + "no right on cat %r" % cat.id + ) + return super().update(filters, attrs) + + def get_history(self, year=None, month=None): + """ + Sort articles by year and month. + """ + articles_counter = Counter() + articles = self.read() + if year is not None: + articles = articles.filter(sqlalchemy.extract("year", Article.date) == year) + if month is not None: + articles = articles.filter( + sqlalchemy.extract("month", Article.date) == month + ) + for article in articles.all(): + if year is not None: + articles_counter[article.date.month] += 1 + else: + articles_counter[article.date.year] += 1 + return articles_counter, articles + + def read_light(self, **filters): + return ( + super() + .read(**filters) + .with_entities( + Article.id, + Article.title, + Article.readed, + Article.like, + Article.feed_id, + Article.date, + Article.category_id, + ) + .order_by(Article.date.desc()) + ) + + def read_ordered(self, **filters): + return super().read(**filters).order_by(Article.date.desc()) diff --git a/newspipe/controllers/bookmark.py b/newspipe/controllers/bookmark.py new file mode 100644 index 00000000..3c9dcce9 --- /dev/null +++ b/newspipe/controllers/bookmark.py @@ -0,0 +1,35 @@ +import logging +import itertools +from datetime import datetime, timedelta + +from newspipe.bootstrap import db +from newspipe.models import Bookmark +from .abstract import AbstractController +from .tag import BookmarkTagController + +logger = logging.getLogger(__name__) + + +class BookmarkController(AbstractController): + _db_cls = Bookmark + + def count_by_href(self, **filters): + return self._count_by(Bookmark.href, filters) + + def update(self, filters, attrs): + BookmarkTagController(self.user_id).read( + **{"bookmark_id": filters["id"]} + ).delete() + + for tag in attrs["tags"]: + BookmarkTagController(self.user_id).create( + **{ + "text": tag.text, + "id": tag.id, + "bookmark_id": tag.bookmark_id, + "user_id": tag.user_id, + } + ) + + del attrs["tags"] + return super().update(filters, attrs) diff --git a/newspipe/controllers/category.py b/newspipe/controllers/category.py new file mode 100644 index 00000000..36f1dea2 --- /dev/null +++ b/newspipe/controllers/category.py @@ -0,0 +1,13 @@ +from .abstract import AbstractController +from newspipe.models import Category +from .feed import FeedController + + +class CategoryController(AbstractController): + _db_cls = Category + + def delete(self, obj_id): + FeedController(self.user_id).update( + {"category_id": obj_id}, {"category_id": None} + ) + return super().delete(obj_id) diff --git a/newspipe/controllers/feed.py b/newspipe/controllers/feed.py new file mode 100644 index 00000000..798efa6e --- /dev/null +++ b/newspipe/controllers/feed.py @@ -0,0 +1,105 @@ +import logging +import itertools +from datetime import datetime, timedelta + +from newspipe.bootstrap import application +from .abstract import AbstractController +from .icon import IconController +from newspipe.models import User, Feed +from newspipe.lib.utils import clear_string + +logger = logging.getLogger(__name__) +DEFAULT_LIMIT = 5 +DEFAULT_MAX_ERROR = application.config['DEFAULT_MAX_ERROR'] + + +class FeedController(AbstractController): + _db_cls = Feed + + def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + return [ + feed + for feed in self.read( + error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last + ) + .join(User) + .filter(User.is_active == True) + .order_by("last_retrieved") + .limit(limit) + ] + + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + now = datetime.now() + max_last = now - timedelta(minutes=60) + feeds = self.list_late(max_last, max_error, limit) + if feeds: + self.update( + {"id__in": [feed.id for feed in feeds]}, {"last_retrieved": now} + ) + return feeds + + def get_duplicates(self, feed_id): + """ + Compare a list of documents by pair. + Pairs of duplicates are sorted by "retrieved date". + """ + feed = self.get(id=feed_id) + duplicates = [] + for pair in itertools.combinations(feed.articles[:1000], 2): + date1, date2 = pair[0].date, pair[1].date + if clear_string(pair[0].title) == clear_string(pair[1].title) and ( + date1 - date2 + ) < timedelta(days=1): + if pair[0].retrieved_date < pair[1].retrieved_date: + duplicates.append((pair[0], pair[1])) + else: + duplicates.append((pair[1], pair[0])) + return feed, duplicates + + def get_inactives(self, nb_days): + today = datetime.now() + inactives = [] + for feed in self.read(): + try: + last_post = feed.articles[0].date + except IndexError: + continue + except Exception as e: + logger.exception(e) + continue + elapsed = today - last_post + if elapsed > timedelta(days=nb_days): + inactives.append((feed, elapsed)) + inactives.sort(key=lambda tup: tup[1], reverse=True) + return inactives + + def count_by_category(self, **filters): + return self._count_by(Feed.category_id, filters) + + def count_by_link(self, **filters): + return self._count_by(Feed.link, filters) + + def _ensure_icon(self, attrs): + if not attrs.get("icon_url"): + return + icon_contr = IconController() + if not icon_contr.read(url=attrs["icon_url"]).count(): + icon_contr.create(**{"url": attrs["icon_url"]}) + + def create(self, **attrs): + self._ensure_icon(attrs) + return super().create(**attrs) + + def update(self, filters, attrs): + from .article import ArticleController + + self._ensure_icon(attrs) + if "category_id" in attrs and attrs["category_id"] == 0: + del attrs["category_id"] + elif "category_id" in attrs: + art_contr = ArticleController(self.user_id) + for feed in self.read(**filters): + art_contr.update( + {"feed_id": feed.id}, {"category_id": attrs["category_id"]} + ) + return super().update(filters, attrs) diff --git a/newspipe/controllers/icon.py b/newspipe/controllers/icon.py new file mode 100644 index 00000000..0fa7a39e --- /dev/null +++ b/newspipe/controllers/icon.py @@ -0,0 +1,27 @@ +import base64 +import requests +from newspipe.models import Icon +from .abstract import AbstractController + + +class IconController(AbstractController): + _db_cls = Icon + _user_id_key = None + + def _build_from_url(self, attrs): + if "url" in attrs and "content" not in attrs: + resp = requests.get(attrs["url"], verify=False) + attrs.update( + { + "url": resp.url, + "mimetype": resp.headers.get("content-type", None), + "content": base64.b64encode(resp.content).decode("utf8"), + } + ) + return attrs + + def create(self, **attrs): + return super().create(**self._build_from_url(attrs)) + + def update(self, filters, attrs): + return super().update(filters, self._build_from_url(attrs)) diff --git a/newspipe/controllers/tag.py b/newspipe/controllers/tag.py new file mode 100644 index 00000000..c97ce20e --- /dev/null +++ b/newspipe/controllers/tag.py @@ -0,0 +1,22 @@ +import logging +import itertools +from datetime import datetime, timedelta + +from newspipe.bootstrap import db +from .abstract import AbstractController +from newspipe.models.tag import BookmarkTag + +logger = logging.getLogger(__name__) + + +class BookmarkTagController(AbstractController): + _db_cls = BookmarkTag + + def count_by_href(self, **filters): + return self._count_by(BookmarkTag.text, filters) + + def create(self, **attrs): + return super().create(**attrs) + + def update(self, filters, attrs): + return super().update(filters, attrs) diff --git a/newspipe/controllers/user.py b/newspipe/controllers/user.py new file mode 100644 index 00000000..7396e52c --- /dev/null +++ b/newspipe/controllers/user.py @@ -0,0 +1,28 @@ +import logging +from werkzeug.security import generate_password_hash, check_password_hash +from .abstract import AbstractController +from newspipe.models import User + +logger = logging.getLogger(__name__) + + +class UserController(AbstractController): + _db_cls = User + _user_id_key = "id" + + def _handle_password(self, attrs): + if attrs.get("password"): + attrs["pwdhash"] = generate_password_hash(attrs.pop("password")) + elif "password" in attrs: + del attrs["password"] + + def check_password(self, user, password): + return check_password_hash(user.pwdhash, password) + + def create(self, **attrs): + self._handle_password(attrs) + return super().create(**attrs) + + def update(self, filters, attrs): + self._handle_password(attrs) + return super().update(filters, attrs) diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py index 828066ff..9296e5e4 100644 --- a/newspipe/crawler/default_crawler.py +++ b/newspipe/crawler/default_crawler.py @@ -1,8 +1,8 @@ #! /usr/bin/env python # -*- coding: utf-8 - -# newspipe - A Web based news aggregator. -# Copyright (C) 2010-2019 Cédric Bonhomme - https://www.cedricbonhomme.org +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org # # For more information: https://git.sr.ht/~cedric/newspipe # @@ -34,13 +34,13 @@ import dateutil.parser from datetime import datetime, timezone, timedelta from sqlalchemy import or_ -import conf -from bootstrap import db -from web.models import User -from web.controllers import FeedController, ArticleController -from lib.utils import jarr_get -from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.article_utils import construct_article, extract_id, get_article_content +from newspipe.bootstrap import application +from newspipe.bootstrap import db +from newspipe.models import User +from newspipe.controllers import FeedController, ArticleController +from newspipe.lib.utils import newspipe_get +from newspipe.lib.feed_utils import construct_feed_from, is_parsing_ok +from newspipe.lib.article_utils import construct_article, extract_id, get_article_content logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ async def parse_feed(user, feed): # with (await sem): try: logger.info("Retrieving feed {}".format(feed.link)) - resp = await jarr_get(feed.link, timeout=5) + resp = await newspipe_get(feed.link, timeout=5) except Exception as e: logger.info("Problem when reading feed {}".format(feed.link)) return @@ -163,9 +163,9 @@ async def retrieve_feed(queue, users, feed_id=None): if feed_id is not None: filters["id"] = feed_id filters["enabled"] = True - filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] filters["last_retrieved__lt"] = datetime.now() - timedelta( - minutes=conf.FEED_REFRESH_INTERVAL + minutes=application.config['FEED_REFRESH_INTERVAL'] ) feeds = FeedController().read(**filters).all() diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py index c2494c79..77cea397 100644 --- a/newspipe/lib/article_utils.py +++ b/newspipe/lib/article_utils.py @@ -9,8 +9,8 @@ import dateutil.parser from bs4 import BeautifulSoup, SoupStrainer from requests.exceptions import MissingSchema -import conf -from lib.utils import jarr_get +from newspipe.bootstrap import application +from newspipe.lib.utils import newspipe_get logger = logging.getLogger(__name__) PROCESSED_DATE_KEYS = {"published", "created", "updated"} @@ -77,16 +77,16 @@ def get_article_content(entry): async def get_article_details(entry, fetch=True): article_link = entry.get("link") article_title = html.unescape(entry.get("title", "")) - if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: + if fetch and application.config['CRAWLER_RESOLV'] and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) - response = await jarr_get(article_link, timeout=5) + response = await newspipe_get(article_link, timeout=5) except MissingSchema: split, failed = urlsplit(article_link), False for scheme in "https", "http": new_link = urlunsplit(SplitResult(scheme, *split[1:])) try: - response = await jarr_get(new_link, timeout=5) + response = await newspipe_get(new_link, timeout=5) except Exception as error: failed = True continue diff --git a/newspipe/lib/data.py b/newspipe/lib/data.py index b8cc3c07..dd7a0503 100644 --- a/newspipe/lib/data.py +++ b/newspipe/lib/data.py @@ -35,10 +35,10 @@ import opml import datetime from flask import jsonify -from bootstrap import db -from web.models import User, Feed, Article -from web.models.tag import BookmarkTag -from web.controllers import BookmarkController, BookmarkTagController +from newspipe.bootstrap import db +from newspipe.models import User, Feed, Article +from newspipe.models.tag import BookmarkTag +from newspipe.controllers import BookmarkController, BookmarkTagController def import_opml(nickname, opml_content): diff --git a/newspipe/lib/feed_utils.py b/newspipe/lib/feed_utils.py index 9f1e2354..a4c6c275 100644 --- a/newspipe/lib/feed_utils.py +++ b/newspipe/lib/feed_utils.py @@ -3,10 +3,10 @@ import urllib import logging import requests import feedparser -from conf import CRAWLER_USER_AGENT from bs4 import BeautifulSoup, SoupStrainer -from lib.utils import try_keys, try_get_icon_url, rebuild_url +from newspipe.bootstrap import application +from newspipe.lib.utils import try_keys, try_get_icon_url, rebuild_url logger = logging.getLogger(__name__) logging.captureWarnings(True) @@ -39,7 +39,7 @@ def escape_keys(*keys): @escape_keys("title", "description") def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): - requests_kwargs = {"headers": {"User-Agent": CRAWLER_USER_AGENT}, "verify": False} + requests_kwargs = {"headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, "verify": False} if url is None and fp_parsed is not None: url = fp_parsed.get("url") if url is not None and fp_parsed is None: diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py index ab57037f..b1ccd075 100755 --- a/newspipe/lib/misc_utils.py +++ b/newspipe/lib/misc_utils.py @@ -45,9 +45,9 @@ from collections import Counter from contextlib import contextmanager from flask import request -import conf -from web.controllers import ArticleController -from lib.utils import clear_string +from newspipe.bootstrap import application +from newspipe.controllers import ArticleController +from newspipe.lib.utils import clear_string logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def fetch(id, feed_id=None): """ cmd = [ sys.executable, - conf.BASE_DIR + "/manager.py", + application.config['BASE_DIR'] + "/manager.py", "fetch_asyncio", "--user_id=" + str(id), ] @@ -154,7 +154,7 @@ def load_stop_words(): Load the stop words and return them in a list. """ stop_words_lists = glob.glob( - os.path.join(conf.BASE_DIR, "web/var/stop_words/*.txt") + os.path.join(application.config['BASE_DIR'], "web/var/stop_words/*.txt") ) stop_words = [] diff --git a/newspipe/lib/utils.py b/newspipe/lib/utils.py index f7244e17..958c9f23 100644 --- a/newspipe/lib/utils.py +++ b/newspipe/lib/utils.py @@ -6,7 +6,7 @@ import requests from hashlib import md5 from flask import request, url_for -import conf +from newspipe.bootstrap import application logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def try_get_icon_url(url, *splits): response = None # if html in content-type, we assume it's a fancy 404 page try: - response = jarr_get(rb_url) + response = newspipe_get(rb_url) content_type = response.headers.get("content-type", "") except Exception: pass @@ -89,12 +89,12 @@ def redirect_url(default="home"): return request.args.get("next") or request.referrer or url_for(default) -async def jarr_get(url, **kwargs): +async def newspipe_get(url, **kwargs): request_kwargs = { "verify": False, "allow_redirects": True, - "timeout": conf.CRAWLER_TIMEOUT, - "headers": {"User-Agent": conf.CRAWLER_USER_AGENT}, + "timeout": application.config['CRAWLER_TIMEOUT'], + "headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, } request_kwargs.update(kwargs) return requests.get(url, **request_kwargs) diff --git a/newspipe/manager.py b/newspipe/manager.py deleted file mode 100755 index 60f4c729..00000000 --- a/newspipe/manager.py +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import logging -from datetime import datetime -from werkzeug.security import generate_password_hash -from bootstrap import application, db, conf, set_logging -from flask_script import Manager -from flask_migrate import Migrate, MigrateCommand - -import web.models -from web.controllers import UserController - -logger = logging.getLogger("manager") - -Migrate(application, db) - -manager = Manager(application) -manager.add_command("db", MigrateCommand) - - -@manager.command -def db_empty(): - "Will drop every datas stocked in db." - with application.app_context(): - web.models.db_empty(db) - - -@manager.command -def db_create(): - "Will create the database from conf parameters." - admin = { - "is_admin": True, - "is_api": True, - "is_active": True, - "nickname": "admin", - "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")), - } - with application.app_context(): - db.create_all() - UserController(ignore_context=True).create(**admin) - - -@manager.command -def create_admin(nickname, password): - "Will create an admin user." - admin = { - "is_admin": True, - "is_api": True, - "is_active": True, - "nickname": nickname, - "pwdhash": generate_password_hash(password), - } - with application.app_context(): - UserController(ignore_context=True).create(**admin) - - -@manager.command -def fetch_asyncio(user_id=None, feed_id=None): - "Crawl the feeds with asyncio." - import asyncio - - with application.app_context(): - from crawler import default_crawler - - filters = {} - filters["is_active"] = True - filters["automatic_crawling"] = True - if None is not user_id: - filters["id"] = user_id - users = UserController().read(**filters).all() - - try: - feed_id = int(feed_id) - except: - feed_id = None - - loop = asyncio.get_event_loop() - queue = asyncio.Queue(maxsize=3, loop=loop) - - producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) - consumer_coro = default_crawler.insert_articles(queue, 1) - - logger.info("Starting crawler.") - start = datetime.now() - loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) - end = datetime.now() - loop.close() - logger.info("Crawler finished in {} seconds.".format((end - start).seconds)) - - -if __name__ == "__main__": - manager.run() diff --git a/newspipe/models/__init__.py b/newspipe/models/__init__.py new file mode 100644 index 00000000..a58a7ad5 --- /dev/null +++ b/newspipe/models/__init__.py @@ -0,0 +1,98 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from .feed import Feed +from .role import Role +from .user import User +from .article import Article +from .icon import Icon +from .category import Category +from .tag import BookmarkTag +from .tag import ArticleTag +from .bookmark import Bookmark + +__all__ = [ + "Feed", + "Role", + "User", + "Article", + "Icon", + "Category", + "Bookmark", + "ArticleTag", + "BookmarkTag", +] + +import os + +from sqlalchemy.engine import reflection +from sqlalchemy.schema import ( + MetaData, + Table, + DropTable, + ForeignKeyConstraint, + DropConstraint, +) + + +def db_empty(db): + "Will drop every datas stocked in db." + # From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything + conn = db.engine.connect() + + # the transaction only applies if the DB supports + # transactional DDL, i.e. Postgresql, MS SQL Server + trans = conn.begin() + + inspector = reflection.Inspector.from_engine(db.engine) + + # gather all data first before dropping anything. + # some DBs lock after things have been dropped in + # a transaction. + metadata = MetaData() + + tbs = [] + all_fks = [] + + for table_name in inspector.get_table_names(): + fks = [] + for fk in inspector.get_foreign_keys(table_name): + if not fk["name"]: + continue + fks.append(ForeignKeyConstraint((), (), name=fk["name"])) + t = Table(table_name, metadata, *fks) + tbs.append(t) + all_fks.extend(fks) + + for fkc in all_fks: + conn.execute(DropConstraint(fkc)) + + for table in tbs: + conn.execute(DropTable(table)) + + trans.commit() diff --git a/newspipe/models/article.py b/newspipe/models/article.py new file mode 100644 index 00000000..ecc2352b --- /dev/null +++ b/newspipe/models/article.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.5 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2016/10/04 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from newspipe.bootstrap import db +from datetime import datetime +from sqlalchemy import Index +from sqlalchemy.ext.associationproxy import association_proxy + +from newspipe.models.right_mixin import RightMixin + + +class Article(db.Model, RightMixin): + "Represent an article from a feed." + id = db.Column(db.Integer(), primary_key=True) + entry_id = db.Column(db.String(), nullable=False) + link = db.Column(db.String()) + title = db.Column(db.String()) + content = db.Column(db.String()) + readed = db.Column(db.Boolean(), default=False) + like = db.Column(db.Boolean(), default=False) + date = db.Column(db.DateTime(), default=datetime.utcnow) + updated_date = db.Column(db.DateTime(), default=datetime.utcnow) + retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow) + + # foreign keys + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) + feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id")) + category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) + + # relationships + tag_objs = db.relationship( + "ArticleTag", + back_populates="article", + cascade="all,delete-orphan", + lazy=False, + foreign_keys="[ArticleTag.article_id]", + ) + tags = association_proxy("tag_objs", "text") + + # indexes + # __table_args__ = ( + # Index('user_id'), + # Index('user_id', 'category_id'), + # Index('user_id', 'feed_id'), + # Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id) + # ) + + # api whitelists + @staticmethod + def _fields_base_write(): + return {"readed", "like", "feed_id", "category_id"} + + @staticmethod + def _fields_base_read(): + return { + "id", + "entry_id", + "link", + "title", + "content", + "date", + "retrieved_date", + "user_id", + "tags", + } + + @staticmethod + def _fields_api_write(): + return {"tags"} + + def __repr__(self): + return ( + "" + % (self.id, self.entry_id, self.title, self.date, self.retrieved_date) + ) diff --git a/newspipe/models/bookmark.py b/newspipe/models/bookmark.py new file mode 100644 index 00000000..c3bc2dea --- /dev/null +++ b/newspipe/models/bookmark.py @@ -0,0 +1,72 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.1 $" +__date__ = "$Date: 2016/12/07 $" +__revision__ = "$Date: 2016/12/07 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from newspipe.bootstrap import db +from datetime import datetime +from sqlalchemy import desc +from sqlalchemy.orm import validates +from sqlalchemy.ext.associationproxy import association_proxy + +from newspipe.models.tag import BookmarkTag +from newspipe.models.right_mixin import RightMixin + + +class Bookmark(db.Model, RightMixin): + """ + Represent a bookmark. + """ + + id = db.Column(db.Integer(), primary_key=True) + href = db.Column(db.String(), default="") + title = db.Column(db.String(), default="") + description = db.Column(db.String(), default="") + shared = db.Column(db.Boolean(), default=False) + to_read = db.Column(db.Boolean(), default=False) + time = db.Column(db.DateTime(), default=datetime.utcnow) + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) + + # relationships + tags = db.relationship( + BookmarkTag, + backref="of_bookmark", + lazy="dynamic", + cascade="all,delete-orphan", + order_by=desc(BookmarkTag.text), + ) + tags_proxy = association_proxy("tags", "text") + + @validates("description") + def validates_title(self, key, value): + return str(value).strip() + + @validates("extended") + def validates_description(self, key, value): + return str(value).strip() + + def __repr__(self): + return "" % (self.href) diff --git a/newspipe/models/category.py b/newspipe/models/category.py new file mode 100644 index 00000000..3abb6717 --- /dev/null +++ b/newspipe/models/category.py @@ -0,0 +1,28 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +from newspipe.bootstrap import db +from sqlalchemy import Index +from newspipe.models.right_mixin import RightMixin + + +class Category(db.Model, RightMixin): + id = db.Column(db.Integer(), primary_key=True) + name = db.Column(db.String()) + + # relationships + user_id = db.Column(db.Integer, db.ForeignKey("user.id")) + feeds = db.relationship("Feed", cascade="all,delete-orphan") + articles = db.relationship("Article", cascade="all,delete-orphan") + + # index + idx_category_uid = Index("user_id") + + # api whitelists + @staticmethod + def _fields_base_read(): + return {"id", "user_id"} + + @staticmethod + def _fields_base_write(): + return {"name"} diff --git a/newspipe/models/feed.py b/newspipe/models/feed.py new file mode 100644 index 00000000..fc88c17f --- /dev/null +++ b/newspipe/models/feed.py @@ -0,0 +1,105 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from newspipe.bootstrap import db +from datetime import datetime +from sqlalchemy import desc, Index +from sqlalchemy.orm import validates +from newspipe.models.right_mixin import RightMixin +from newspipe.models.article import Article + + +class Feed(db.Model, RightMixin): + """ + Represent a feed. + """ + + id = db.Column(db.Integer(), primary_key=True) + title = db.Column(db.String(), default="") + description = db.Column(db.String(), default="FR") + link = db.Column(db.String(), nullable=False) + site_link = db.Column(db.String(), default="") + enabled = db.Column(db.Boolean(), default=True) + created_date = db.Column(db.DateTime(), default=datetime.utcnow) + filters = db.Column(db.PickleType, default=[]) + private = db.Column(db.Boolean(), default=False) + + # cache handling + etag = db.Column(db.String(), default="") + last_modified = db.Column(db.String(), default="") + last_retrieved = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # error logging + last_error = db.Column(db.String(), default="") + error_count = db.Column(db.Integer(), default=0) + + # relationship + icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None) + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) + category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) + articles = db.relationship( + Article, + backref="source", + lazy="dynamic", + cascade="all,delete-orphan", + order_by=desc(Article.date), + ) + + # index + idx_feed_uid_cid = Index("user_id", "category_id") + idx_feed_uid = Index("user_id") + + # api whitelists + @staticmethod + def _fields_base_write(): + return { + "title", + "description", + "link", + "site_link", + "enabled", + "filters", + "last_error", + "error_count", + "category_id", + } + + @staticmethod + def _fields_base_read(): + return {"id", "user_id", "icon_url", "last_retrieved"} + + @validates("title") + def validates_title(self, key, value): + return str(value).strip() + + @validates("description") + def validates_description(self, key, value): + return str(value).strip() + + def __repr__(self): + return "" % (self.title) diff --git a/newspipe/models/icon.py b/newspipe/models/icon.py new file mode 100644 index 00000000..99c10224 --- /dev/null +++ b/newspipe/models/icon.py @@ -0,0 +1,10 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +from newspipe.bootstrap import db + + +class Icon(db.Model): + url = db.Column(db.String(), primary_key=True) + content = db.Column(db.String(), default=None) + mimetype = db.Column(db.String(), default="application/image") diff --git a/newspipe/models/right_mixin.py b/newspipe/models/right_mixin.py new file mode 100644 index 00000000..670beafa --- /dev/null +++ b/newspipe/models/right_mixin.py @@ -0,0 +1,64 @@ +from sqlalchemy.ext.associationproxy import _AssociationList + + +class RightMixin: + @staticmethod + def _fields_base_write(): + return set() + + @staticmethod + def _fields_base_read(): + return set(["id"]) + + @staticmethod + def _fields_api_write(): + return set([]) + + @staticmethod + def _fields_api_read(): + return set(["id"]) + + @classmethod + def fields_base_write(cls): + return cls._fields_base_write() + + @classmethod + def fields_base_read(cls): + return cls._fields_base_write().union(cls._fields_base_read()) + + @classmethod + def fields_api_write(cls): + return cls.fields_base_write().union(cls._fields_api_write()) + + @classmethod + def fields_api_read(cls): + return cls.fields_base_read().union(cls._fields_api_read()) + + def __getitem__(self, key): + if not hasattr(self, "__dump__"): + self.__dump__ = {} + return self.__dump__.get(key) + + def __setitem__(self, key, value): + if not hasattr(self, "__dump__"): + self.__dump__ = {} + self.__dump__[key] = value + + def dump(self, role="admin"): + if role == "admin": + dico = { + k: getattr(self, k) + for k in set(self.__table__.columns.keys()) + .union(self.fields_api_read()) + .union(self.fields_base_read()) + } + elif role == "api": + dico = {k: getattr(self, k) for k in self.fields_api_read()} + else: + dico = {k: getattr(self, k) for k in self.fields_base_read()} + if hasattr(self, "__dump__"): + dico.update(self.__dump__) + for key, value in dico.items(): # preventing association proxy to die + if isinstance(value, _AssociationList): + dico[key] = list(value) + return dico diff --git a/newspipe/models/role.py b/newspipe/models/role.py new file mode 100644 index 00000000..628467c0 --- /dev/null +++ b/newspipe/models/role.py @@ -0,0 +1,40 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from newspipe.bootstrap import db + + +class Role(db.Model): + """ + Represent a role. + """ + + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(), unique=True) + + user_id = db.Column(db.Integer, db.ForeignKey("user.id")) diff --git a/newspipe/models/tag.py b/newspipe/models/tag.py new file mode 100644 index 00000000..b853c344 --- /dev/null +++ b/newspipe/models/tag.py @@ -0,0 +1,44 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +from newspipe.bootstrap import db + + +class ArticleTag(db.Model): + text = db.Column(db.String, primary_key=True, unique=False) + + # foreign keys + article_id = db.Column( + db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True + ) + + # relationships + article = db.relationship( + "Article", back_populates="tag_objs", foreign_keys=[article_id] + ) + + def __init__(self, text): + self.text = text + + +class BookmarkTag(db.Model): + id = db.Column(db.Integer, primary_key=True) + text = db.Column(db.String, unique=False) + + # foreign keys + user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE")) + bookmark_id = db.Column( + db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE") + ) + + # relationships + bookmark = db.relationship( + "Bookmark", + back_populates="tags", + cascade="all,delete", + foreign_keys=[bookmark_id], + ) + + # def __init__(self, text, user_id): + # self.text = text + # self.user_id = user_id diff --git a/newspipe/models/user.py b/newspipe/models/user.py new file mode 100644 index 00000000..142536fe --- /dev/null +++ b/newspipe/models/user.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +import re +import random +import hashlib +from datetime import datetime +from werkzeug.security import check_password_hash +from flask_login import UserMixin +from sqlalchemy.orm import validates + +from newspipe.bootstrap import db +from newspipe.models.right_mixin import RightMixin +from newspipe.models.category import Category +from newspipe.models.feed import Feed + + +class User(db.Model, UserMixin, RightMixin): + """ + Represent a user. + """ + + id = db.Column(db.Integer, primary_key=True) + nickname = db.Column(db.String(), unique=True) + pwdhash = db.Column(db.String()) + + automatic_crawling = db.Column(db.Boolean(), default=True) + + is_public_profile = db.Column(db.Boolean(), default=False) + bio = db.Column(db.String(5000), default="") + webpage = db.Column(db.String(), default="") + twitter = db.Column(db.String(), default="") + + date_created = db.Column(db.DateTime(), default=datetime.utcnow) + last_seen = db.Column(db.DateTime(), default=datetime.utcnow) + + # user rights + is_active = db.Column(db.Boolean(), default=False) + is_admin = db.Column(db.Boolean(), default=False) + is_api = db.Column(db.Boolean(), default=False) + + # relationships + categories = db.relationship( + "Category", + backref="user", + cascade="all, delete-orphan", + foreign_keys=[Category.user_id], + ) + feeds = db.relationship( + "Feed", + backref="user", + cascade="all, delete-orphan", + foreign_keys=[Feed.user_id], + ) + + @staticmethod + def _fields_base_write(): + return {"login", "password"} + + @staticmethod + def _fields_base_read(): + return {"date_created", "last_connection"} + + @staticmethod + def make_valid_nickname(nickname): + return re.sub("[^a-zA-Z0-9_\.]", "", nickname) + + @validates("bio") + def validates_bio(self, key, value): + assert len(value) <= 5000, AssertionError("maximum length for bio: 5000") + return value.strip() + + def get_id(self): + """ + Return the id of the user. + """ + return self.id + + def check_password(self, password): + """ + Check the password of the user. + """ + return check_password_hash(self.pwdhash, password) + + def __eq__(self, other): + return self.id == other.id + + def __repr__(self): + return "" % (self.nickname) diff --git a/newspipe/notifications/emails.py b/newspipe/notifications/emails.py index dcfcf771..9738ae95 100644 --- a/newspipe/notifications/emails.py +++ b/newspipe/notifications/emails.py @@ -24,8 +24,8 @@ import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -import conf -from web.decorators import async_maker +from newspipe.bootstrap import application +from newspipe.web.decorators import async_maker logger = logging.getLogger(__name__) @@ -33,8 +33,8 @@ logger = logging.getLogger(__name__) @async_maker def send_async_email(mfrom, mto, msg): try: - s = smtplib.SMTP(conf.NOTIFICATION_HOST) - s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) + s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) + s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) except Exception: logger.exception("send_async_email raised:") else: @@ -56,7 +56,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): # Create message container - the correct MIME type is multipart/alternative. msg = MIMEMultipart("alternative") msg["Subject"] = subject - msg["From"] = conf.NOTIFICATION_EMAIL + msg["From"] = application.config['NOTIFICATION_EMAIL'] msg["To"] = to msg["BCC"] = bcc @@ -71,12 +71,12 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): msg.attach(part2) try: - s = smtplib.SMTP(conf.NOTIFICATION_HOST) - s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) + s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) + s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) except Exception: logger.exception("send_smtp raised:") else: s.sendmail( - conf.NOTIFICATION_EMAIL, msg["To"] + ", " + msg["BCC"], msg.as_string() + application.config['NOTIFICATION_EMAIL'], msg["To"] + ", " + msg["BCC"], msg.as_string() ) s.quit() diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py index 664c19a7..71ab0cf8 100644 --- a/newspipe/notifications/notifications.py +++ b/newspipe/notifications/notifications.py @@ -21,9 +21,10 @@ import datetime from flask import render_template -import conf -from notifications import emails -from web.lib.user_utils import generate_confirmation_token + +from newspipe.bootstrap import application +from newspipe.notifications import emails +from newspipe.web.lib.user_utils import generate_confirmation_token def new_account_notification(user, email): @@ -32,20 +33,20 @@ def new_account_notification(user, email): """ token = generate_confirmation_token(user.nickname) expire_time = datetime.datetime.now() + datetime.timedelta( - seconds=conf.TOKEN_VALIDITY_PERIOD + seconds=application.config['TOKEN_VALIDITY_PERIOD'] ) plaintext = render_template( "emails/account_activation.txt", user=user, - platform_url=conf.PLATFORM_URL, + platform_url=application.config['PLATFORM_URL'], token=token, expire_time=expire_time, ) emails.send( to=email, - bcc=conf.NOTIFICATION_EMAIL, + bcc=application.config['NOTIFICATION_EMAIL'], subject="[Newspipe] Account creation", plaintext=plaintext, ) @@ -58,7 +59,7 @@ def new_password_notification(user, password): plaintext = render_template("emails/new_password.txt", user=user, password=password) emails.send( to=user.email, - bcc=conf.NOTIFICATION_EMAIL, + bcc=application.config['NOTIFICATION_EMAIL'], subject="[Newspipe] New password", plaintext=plaintext, ) diff --git a/newspipe/runserver.py b/newspipe/runserver.py deleted file mode 100755 index 8880d5b9..00000000 --- a/newspipe/runserver.py +++ /dev/null @@ -1,66 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# Newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -import calendar -from bootstrap import conf, application, populate_g -from flask_babel import Babel, format_datetime - - -babel = Babel(application) - - -# Jinja filters -def month_name(month_number): - return calendar.month_name[month_number] - - -application.jinja_env.filters["month_name"] = month_name -application.jinja_env.filters["datetime"] = format_datetime -application.jinja_env.globals["conf"] = conf - -# Views -from flask_restful import Api -from flask import g - -with application.app_context(): - populate_g() - g.api = Api(application, prefix="/api/v2.0") - g.babel = babel - - from web import views - - application.register_blueprint(views.articles_bp) - application.register_blueprint(views.article_bp) - application.register_blueprint(views.feeds_bp) - application.register_blueprint(views.feed_bp) - application.register_blueprint(views.categories_bp) - application.register_blueprint(views.category_bp) - application.register_blueprint(views.icon_bp) - application.register_blueprint(views.admin_bp) - application.register_blueprint(views.users_bp) - application.register_blueprint(views.user_bp) - application.register_blueprint(views.bookmarks_bp) - application.register_blueprint(views.bookmark_bp) - - -if __name__ == "__main__": - application.run( - host=conf.WEBSERVER_HOST, port=conf.WEBSERVER_PORT, debug=conf.WEBSERVER_DEBUG - ) diff --git a/newspipe/static/css/custom.css b/newspipe/static/css/custom.css new file mode 100644 index 00000000..c85cc8a1 --- /dev/null +++ b/newspipe/static/css/custom.css @@ -0,0 +1,80 @@ +html { + position: relative; + min-height: 100%; + font-size: 16px; +} +body { + /* Margin bottom by footer height */ + margin-bottom: 60px; + background-color: rgb(246, 248, 250); +} +img { + padding: 2px; +} +a { + color: #3572B0; +} + +#sidebar { + overflow: auto; + max-height: 700px; +} + +#sidebar .nav li { + list-style-type: none; + padding: 0; + line-height: 1.0; + flex-direction: row; + flex-wrap: nowrap; /* assumes you only want one row */ +} + +.navbar-dark .navbar-nav .nav-link { + color: + rgb(255, 255, 255, 1); +} + +.bg-newspipe-blue { + /* background-color: #0082c9; */ + background-image: linear-gradient(40deg, #0082c9 0%, #30b6ff 100%); +} +.btn-primary { + background-color: #0082c9; + opacity: 1; + color: white; +} + +.btn-outline-primary { + border-color: #0082c9; + color: #0082c9; +} + +/* Sticky footer */ +.footer { + position: absolute; + bottom: 0; + width: 100%; + /* Set the fixed height of the footer here */ + height: 60px; + line-height: 60px; /* Vertically center the text there */ + background-color: #006FBA; +} +.footer a{ + color: #FFFFFF; +} + +.input-group-inline { + min-width: 0; + width: 200px; + display: inline; +} + +.alert-message { + position: relative; + display: block; + z-index: 1001; +} + +/*customization for Flask-Paginate*/ +.pagination-page-info { + display: inline; +} diff --git a/newspipe/static/img/favicon.ico b/newspipe/static/img/favicon.ico new file mode 100644 index 00000000..5b056c1e Binary files /dev/null and b/newspipe/static/img/favicon.ico differ diff --git a/newspipe/static/img/newspipe.png b/newspipe/static/img/newspipe.png new file mode 100644 index 00000000..c3ba5029 Binary files /dev/null and b/newspipe/static/img/newspipe.png differ diff --git a/newspipe/static/img/newspipe.svg b/newspipe/static/img/newspipe.svg new file mode 100644 index 00000000..be22ae42 --- /dev/null +++ b/newspipe/static/img/newspipe.svg @@ -0,0 +1,84 @@ + + + + +Created by potrace 1.13, written by Peter Selinger 2001-2015 + + + + + + + + + + + + + + + + + + + diff --git a/newspipe/static/img/pinboard.png b/newspipe/static/img/pinboard.png new file mode 100644 index 00000000..6dddc10b Binary files /dev/null and b/newspipe/static/img/pinboard.png differ diff --git a/newspipe/static/img/reddit.png b/newspipe/static/img/reddit.png new file mode 100755 index 00000000..2d615f2a Binary files /dev/null and b/newspipe/static/img/reddit.png differ diff --git a/newspipe/static/img/twitter.png b/newspipe/static/img/twitter.png new file mode 100644 index 00000000..fc11c4ce Binary files /dev/null and b/newspipe/static/img/twitter.png differ diff --git a/newspipe/static/js/articles.js b/newspipe/static/js/articles.js new file mode 100644 index 00000000..de579532 --- /dev/null +++ b/newspipe/static/js/articles.js @@ -0,0 +1,191 @@ +/*! +* Newspipe - A Web based news aggregator. +* Copyright (C) 2010-2020 Cédric Bonhomme - https://cedricbonhomme.org +* +* For more information: https://git.sr.ht/~cedric/newspipe +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as +* published by the Free Software Foundation, either version 3 of the +* License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . + */ + +API_ROOT = '/api/v2.0/' + +if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } + +function change_unread_counter(feed_id, increment) { + var new_value = parseInt($("#unread-"+feed_id).text()) + increment; + $("#unread-"+feed_id).text(new_value); + $("#total-unread").text(parseInt($("#total-unread").text()) + increment); + if (new_value == 0) { + $("#unread-"+feed_id).hide(); + } else { + $("#unread-"+feed_id).show(); + } +} + ++function ($) { + + // Mark an article as read when it is opened in a new table + $('.open-article').on('click', function(e) { + var feed_id = $(this).parent().parent().attr("data-feed"); + var filter = $('#filters').attr("data-filter"); + if (filter == "unread") { + $(this).parent().parent().remove(); + change_unread_counter(feed_id, -1); + } + }); + + + + // Mark an article as read or unread. + $('.readed').on('click', function() { + var article_id = $(this).parent().parent().parent().attr("data-article"); + var feed_id = $(this).parent().parent().parent().attr("data-feed"); + var filter = $('#filters').attr("data-filter"); + + var data; + if ($(this).hasClass('fa-square-o')) { + data = JSON.stringify({ + readed: false + }) + if (filter == "read") { + $(this).parent().parent().parent().remove(); + } + else { + // here, filter == "all" + $(this).parent().parent().parent().children("td:nth-child(2)").css( "font-weight", "bold" ); + $(this).removeClass('fa-square-o').addClass('fa-check-square-o'); + } + change_unread_counter(feed_id, 1); + } + else { + data = JSON.stringify({readed: true}) + if (filter == "unread") { + $(this).parent().parent().parent().remove(); + } + else { + // here, filter == "all" + $(this).parent().parent().parent().children("td:nth-child(2)").css( "font-weight", "normal" ); + $(this).removeClass('fa-check-square-o').addClass('fa-square-o'); + } + change_unread_counter(feed_id, -1); + } + + // sends the updates to the server + $.ajax({ + type: 'PUT', + // Provide correct Content-Type, so that Flask will know how to process it. + contentType: 'application/json', + // Encode your data as JSON. + data: data, + // This is the type of data you're expecting back from the server. + url: API_ROOT + "article/" + article_id, + success: function (result) { + //console.log(result); + }, + error: function(XMLHttpRequest, textStatus, errorThrown){ + console.log(XMLHttpRequest.responseText); + } + }); + }); + + + + // Like or unlike an article + $('.like').on('click', function() { + var article_id = $(this).parent().parent().parent().attr("data-article"); + var data; + if ($(this).hasClass("glyphicon-star")) { + data = JSON.stringify({like: false}) + $(this).removeClass('glyphicon-star').addClass('glyphicon-star-empty'); + if(window.location.pathname.indexOf('/favorites') != -1) { + $(this).parent().parent().parent().remove(); + } + } + else { + data = JSON.stringify({like: true}) + $(this).removeClass('glyphicon-star-empty').addClass('glyphicon-star'); + } + + // sends the updates to the server + $.ajax({ + type: 'PUT', + // Provide correct Content-Type, so that Flask will know how to process it. + contentType: 'application/json', + // Encode your data as JSON. + data: data, + // This is the type of data you're expecting back from the server. + url: API_ROOT + "article/" + article_id, + success: function (result) { + //console.log(result); + }, + error: function(XMLHttpRequest, textStatus, errorThrown){ + console.log(XMLHttpRequest.responseText); + } + }); + }); + + + + // Delete an article + $('.delete').on('click', function() { + var feed_id = $(this).parent().parent().parent().attr("data-feed"); + var article_id = $(this).parent().parent().parent().attr("data-article"); + $(this).parent().parent().parent().remove(); + + // sends the updates to the server + $.ajax({ + type: 'DELETE', + url: API_ROOT + "article/" + article_id, + success: function (result) { + change_unread_counter(feed_id, -1); + }, + error: function(XMLHttpRequest, textStatus, errorThrown){ + console.log(XMLHttpRequest.responseText); + } + }); + }); + + + + // Delete all duplicate articles (used in the page /duplicates) + $('.delete-all').click(function(){ + var data = []; + + var columnNo = $(this).parent().index(); + $(this).closest("table") + .find("tr td:nth-child(" + (columnNo+1) + ")") + .each(function(line, column) { + data.push(parseInt(column.id)); + }).remove(); + + data = JSON.stringify(data); + + // sends the updates to the server + $.ajax({ + type: 'DELETE', + // Provide correct Content-Type, so that Flask will know how to process it. + contentType: 'application/json', + data: data, + url: API_ROOT + "articles", + success: function (result) { + //console.log(result); + }, + error: function(XMLHttpRequest, textStatus, errorThrown){ + console.log(XMLHttpRequest.responseText); + } + }); + + }); + +}(jQuery); diff --git a/newspipe/static/js/feed.js b/newspipe/static/js/feed.js new file mode 100644 index 00000000..ceef58fc --- /dev/null +++ b/newspipe/static/js/feed.js @@ -0,0 +1,22 @@ +$('.container').on('click', '#add-feed-filter-row', function() { + $('#filters-container').append( + '
' + + ' ' + + ' ' + + ' ' + + ' ' + + ' ' + + '
'); +}); +$('.container').on('click', '.del-feed-filter-row', function() { + $(this).parent().remove(); +}); diff --git a/newspipe/templates/about.html b/newspipe/templates/about.html new file mode 100644 index 00000000..43d8c73d --- /dev/null +++ b/newspipe/templates/about.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+

{{ _('About') }}

+

+ {{ _('Newspipe is a news aggregator platform.') }} +

{{ _('This software is under AGPLv3 license. You are welcome to copy, modify or + redistribute the source code + according to the Affero GPL license.') }}

+

{{ _('Found a bug? Report it here.') }}

+

{{ _('More information') }} {{ _('about this instance.') }}

+
+
+
+
+
+

{{ _('Help') }}

+

{{ _('Contact')}}: {{ contact }}

+

{{ _('You can subscribe to new feeds with a bookmarklet. Drag the following button to your browser bookmarks.') }}

+ {{ _('Subscribe to this feed using Newspipe', bookmarklet='javascript:window.location="%s?url="+encodeURIComponent(document.location)' % url_for('feed.bookmarklet', _external=True)) }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/about_more.html b/newspipe/templates/about_more.html new file mode 100644 index 00000000..94a01c07 --- /dev/null +++ b/newspipe/templates/about_more.html @@ -0,0 +1,11 @@ +{% extends "layout.html" %} +{% block content %} +
+
    +
  • {{ _('Newspipe version') }}: {{newspipe_version}}
    • +
  • {{ _('Registration') }}: {{registration}}
    • +
  • {{ _('Python version') }}: {{python_version}}
    • +
  • {{ _('Number of users') }}: {{nb_users}}
    • +
+
+{% endblock %} diff --git a/newspipe/templates/admin/create_user.html b/newspipe/templates/admin/create_user.html new file mode 100644 index 00000000..8844f987 --- /dev/null +++ b/newspipe/templates/admin/create_user.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% block head%} +{{super()}} +{% endblock %} +{% block content %} +
+
+

{{ message | safe }}

+
+ {{ form.hidden_tag() }} + + {{ form.nickname.label }} + {{ form.nickname(class_="form-control") }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} + + {{ form.password.label }} + {{ form.password(class_="form-control") }} {% for error in form.password.errors %} {{ error }}
{% endfor %} + + {{ form.automatic_crawling.label }} + {{ form.automatic_crawling(class_="form-control") }} {% for error in form.automatic_crawling.errors %} {{ error }}
{% endfor %} + +
+ {{ form.submit(class_="btn btn-primary") }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/admin/dashboard.html b/newspipe/templates/admin/dashboard.html new file mode 100644 index 00000000..8bcc2db0 --- /dev/null +++ b/newspipe/templates/admin/dashboard.html @@ -0,0 +1,68 @@ +{% extends "layout.html" %} +{% block head%} +{{super()}} +{% endblock %} +{% block content %} +
+

{{ _('Registered users') }}

+ + + + + + + + + + + + {% for user in users %} + + + + + + + + {% endfor %} + +
#{{ _('Nickname') }}{{ _('Member since') }}{{ _('Last seen') }}{{ _('Actions') }}
{{ loop.index }} + {% if user.is_public_profile %} + {{ user.nickname }} + {% else %} + {{ user.nickname }} + {% endif %} + {% if user.id == current_user.id %} (It's you!){% endif %} + {{ user.date_created | datetime }}{{ user.last_seen | datetime }} + + {% if user.id != current_user.id %} + + {% if user.is_active %} + + {% else %} + + {% endif %} + + + {% endif %} +
+{{ _('Add a new user') }} +
+ +{% endblock %} diff --git a/newspipe/templates/article.html b/newspipe/templates/article.html new file mode 100644 index 00000000..884bf677 --- /dev/null +++ b/newspipe/templates/article.html @@ -0,0 +1,35 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ article.title|safe }}

+

{{ _('from') }} {{ article.source.title }}

+ + {% if article.like %} + + {% else %} + + {% endif %} + {% if article.readed %} + + {% else %} + + {% endif %} +
{{ article.date | datetime }}
+
+
+ {{ article.content | safe }} +
+
+ + + + + + + + + +
+
+{% endblock %} diff --git a/newspipe/templates/article_pub.html b/newspipe/templates/article_pub.html new file mode 100644 index 00000000..e810d18f --- /dev/null +++ b/newspipe/templates/article_pub.html @@ -0,0 +1,24 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ article.title|safe }}

+

{{ _('from') }} {{ article.source.title }}

+
{{ article.date | datetime }}
+
+
+ {{ article.content | safe }} +
+
+ + + + + + + + + +
+
+{% endblock %} diff --git a/newspipe/templates/bookmarks.html b/newspipe/templates/bookmarks.html new file mode 100644 index 00000000..0f6e02c4 --- /dev/null +++ b/newspipe/templates/bookmarks.html @@ -0,0 +1,81 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+ {{ pagination.info }} +
+
+ {% if current_user.is_authenticated %} + all ⸱  + private ⸱  + public ⸱  + unread + {% endif %} +
+
+
+
+
+ {% if tag %} +   {{ tag }} + {% endif %} + {% if query %} + {% if tag %}
{% endif %} +   {{ query }} + {% endif %} +
+
+
+
+
+
+ +
+ +
+
+
+
+
+
+
+
+
+
+ {{ pagination.links }} +
+
+ +
+
+
    + {% for bookmark in bookmarks %} +
  • + +

    + {{ bookmark.title }} +

    +

    +

    {{ bookmark.description }}
    +
    {% for tag in bookmark.tags %}{{ tag.text }} {% endfor %}
    + {{ bookmark.time | datetime }} + {% if current_user.is_authenticated %} + edit + delete + {% endif %} +

    + +
    • + {% endfor %} +
+
+
+
+
+
+ {{ pagination.links }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/categories.html b/newspipe/templates/categories.html new file mode 100644 index 00000000..ea5388a3 --- /dev/null +++ b/newspipe/templates/categories.html @@ -0,0 +1,36 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ _("You have %(categories)d categories · Add a %(start_link)scategory%(end_link)s", categories=categories|count, start_link=("" % url_for("category.form"))|safe, end_link=""|safe) }}

+ {% if categories|count == 0 %} +

{{_("No category")}}

+ {% else %} +
+ + + + + + + + + + + + {% for category in categories %} + + + + + + + + {% endfor %} + +
#{{ _('Name') }}{{ _('Feeds') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }}{{ category.name }}{{ feeds_count.get(category.id, 0) }}( {{ unread_article_count.get(category.id, 0) }} ) {{ article_count.get(category.id, 0) }} + + +
+
+ {% endif %} +{% endblock %} diff --git a/newspipe/templates/duplicates.html b/newspipe/templates/duplicates.html new file mode 100644 index 00000000..38dc52b1 --- /dev/null +++ b/newspipe/templates/duplicates.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ _('Duplicates in the feed') }} {{ feed.title }}.

+

+ + + + + + + + + + {% for pair in duplicates %} + + + + + + {% endfor %} + +
# + {{ _('Delete all in this column') }} + + {{ _('Delete all in this column') }} +
{{ loop.index }} {{ pair[0].title }} ({{ pair[0].retrieved_date }}) {{ pair[1].title }} ({{ pair[1].retrieved_date }})
+
+
+{% endblock %} diff --git a/newspipe/templates/edit_bookmark.html b/newspipe/templates/edit_bookmark.html new file mode 100644 index 00000000..ee0e0243 --- /dev/null +++ b/newspipe/templates/edit_bookmark.html @@ -0,0 +1,84 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ action }}

+
+ {{ form.hidden_tag() }} +
+ +
+ {{ form.href(class_="form-control", size="100%") }} +
+ {% for error in form.href.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.title(class_="form-control", size="100%") }} +
+ {% for error in form.title.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.description(class_="form-control", size="100%") }} +
+ {% for error in form.description.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.tags(class_="form-control", size="100%") }} +
+ {% for error in form.tags.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+
+ {{ form.shared(class_="checkbox", style="margin-left: 0px;") }} +
+
+
+ +
+ +
+
+ {{ form.to_read(class_="checkbox", style="margin-left: 0px;") }} +
+
+
+ +
+
+ {{ form.submit(class_="btn btn-primary") }} +
+
+
+
+ {% if action == _('Add a new bookmark') %} +
+
+

{{ _('You can add a bookmark with a bookmarklet. Drag the following button to your browser bookmarks.') }}

+ {{ _('Bookmark this page using Newspipe', bookmarklet='javascript:window.location="%s?href="+encodeURIComponent(document.location)+"&title="+document.title' % url_for('bookmark.bookmarklet', _external=True)) }} +
+
+
+

{{ _('Import bookmarks from Pinboard') }} (*.json)

+ + +
+ + +
+
+
+ {% endif %} +
+{% endblock %} diff --git a/newspipe/templates/edit_category.html b/newspipe/templates/edit_category.html new file mode 100644 index 00000000..955d17b9 --- /dev/null +++ b/newspipe/templates/edit_category.html @@ -0,0 +1,23 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ action }}

+
+ {{ form.hidden_tag() }} +
+ +
+ {{ form.name(class_="form-control", size="100%") }} +
+ {% for error in form.name.errors %} {{ error }}
{% endfor %} +
+
+
+ {{ form.submit(class_="btn btn-primary") }} +
+
+
+
+
+{% endblock %} diff --git a/newspipe/templates/edit_feed.html b/newspipe/templates/edit_feed.html new file mode 100644 index 00000000..a439c78d --- /dev/null +++ b/newspipe/templates/edit_feed.html @@ -0,0 +1,98 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ action }}

+
+ {{ form.hidden_tag() }} +
+ +
+ {{ form.link(class_="form-control", size="100%") }} +
+ {% for error in form.link.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.title(class_="form-control", size="100%", placeholder=_('Optional')) }} +
+ {% for error in form.title.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.site_link(class_="form-control", size="100%", placeholder=_('Optional')) }} +
+ {% for error in form.site_link.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+ {{ form.category_id(class_="form-control", placeholder=_('Optional')) }} +
+ {% for error in form.category_id.errors %} {{ error }}
{% endfor %} +
+ +
+ +
+
+ {{ form.enabled(class_="checkbox", style="margin-left: 0px;") }} +
+
+
+ +
+ +
+
+ {{ form.private(class_="checkbox", style="margin-left: 0px;") }} +
+ {{ _("If checked, articles of this feed won't be available to others and the feed won't be listed on your profile page.", url=url_for('user.profile_public', nickname=current_user.nickname) ) }} + {{ _("Check this box if there is a private token in the link of the feed.") }} +
+ +
+ +
+ +
+ +
+
+
+ {% if feed %} + {% for filter_ in feed.filters or [] %} +
+ + + + + +
+ {% endfor %} + {% endif %} +
+ +
+
+ {{ form.submit(class_="btn btn-primary") }} +
+
+
+
+
+{% endblock %} diff --git a/newspipe/templates/emails/account_activation.txt b/newspipe/templates/emails/account_activation.txt new file mode 100644 index 00000000..c7d9c52e --- /dev/null +++ b/newspipe/templates/emails/account_activation.txt @@ -0,0 +1,9 @@ +Hello {{ user.nickname }}, + +Your account has been created. +Click on the following link in order to confirm it: +{{ platform_url }}user/confirm_account/{{ token }} + +The link expires at {{ expire_time.strftime('%Y-%m-%d %H:%M') }}. + +See you, diff --git a/newspipe/templates/emails/new_password.txt b/newspipe/templates/emails/new_password.txt new file mode 100644 index 00000000..1a04a36d --- /dev/null +++ b/newspipe/templates/emails/new_password.txt @@ -0,0 +1,8 @@ +Hello {{ user.nickname }}, + +A new password has been generated at your request: +{{ password }} + +It is advised to replace it as soon as connected to Newspipe. + +See you, diff --git a/newspipe/templates/errors/404.html b/newspipe/templates/errors/404.html new file mode 100644 index 00000000..c64a2be8 --- /dev/null +++ b/newspipe/templates/errors/404.html @@ -0,0 +1,12 @@ +{% extends "layout.html" %} +{% block head %} +{{ super() }} +{% endblock %} +{% block content %} +
+
+

Page Not Found

+

What you were looking for is just not there, go to the home page.

+
+
+{% endblock %} diff --git a/newspipe/templates/errors/500.html b/newspipe/templates/errors/500.html new file mode 100644 index 00000000..417fc0c7 --- /dev/null +++ b/newspipe/templates/errors/500.html @@ -0,0 +1,12 @@ +{% extends "layout.html" %} +{% block head %} +{{ super() }} +{% endblock %} +{% block content %} +
+
+

Internal Server Error

+

Something bad just happened! Go to the home page.

+
+
+{% endblock %} diff --git a/newspipe/templates/feed.html b/newspipe/templates/feed.html new file mode 100644 index 00000000..4246669b --- /dev/null +++ b/newspipe/templates/feed.html @@ -0,0 +1,76 @@ +{% extends "layout.html" %} +{% block content %} +
+
+

{{ feed.title }}

+ {% if feed.description %}

{{ feed.description }}

{% endif %} + {% if current_user.is_authenticated %} + + + {% endif %} +
+
+

+ {{ _('This feed contains') }} {{ feed.articles.all()|count }} {{ _('articles') }}.
+ {% if category %} + {{ _('This feed is part of category %(category_name)s', category_name=category.name) }}
+ {% endif %} + {{ _('Address of the feed') }}: {{ feed.link }}
+ {% if feed.site_link != "" %} + {{ _('Address of the site') }}: {{ feed.site_link }}
+ {% endif %} + +
+ + {% if feed.last_retrieved %} + {{ _("Last download:") }} {{ feed.last_retrieved | datetime }}
+ {% endif %} + + {% if feed.error_count >= conf.DEFAULT_MAX_ERROR %} + {{ _("That feed has encountered too much consecutive errors and won't be retrieved anymore.") }}
+ {{ _("You can click here to reset the error count and reactivate the feed.", reset_error_url=url_for("feed.reset_errors", feed_id=feed.id)) }} + {% elif feed.error_count > 0 %} + {{ _("The download of this feed has encountered some problems. However its error counter will be reinitialized at the next successful retrieving.") }}
+ {% endif %} + + {% if feed.last_error %} + {{ _("Here's the last error encountered while retrieving this feed:") }} 

{{ feed.last_error }}

+ {% endif %} + + {% if feed.articles.all()|count != 0 %} + {{ _('The last article was posted') }} {{ elapsed.days }} {{ _('day(s) ago.') }}
+ {{ _('Daily average') }}: {{ average }}, {{ _('between the') }} {{ first_post_date | datetime }} {{ _('and the') }} {{ end_post_date | datetime }}. + {% endif %} +

+
+ +
+
+
+ + + + + + + + + {% for article in articles %} + + + + + {% endfor %} + +
{{ _('Article') }}{{ _('Date') }}
{{ article.title }}{{ article.date | datetime }}
+
+
+
+ +
+
+ {{ pagination.links }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/feed_list.html b/newspipe/templates/feed_list.html new file mode 100644 index 00000000..2ce7841f --- /dev/null +++ b/newspipe/templates/feed_list.html @@ -0,0 +1,55 @@ +{% if feeds.count() != 0 %} +
+ + + + + + + + + + + + + {% for feed in feeds %} + + + + + + + + + {% endfor %} + +
#{{ _('Status') }}{{ _('Title') }}{{ _('Site') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }} + {% if feed.enabled %} + + {% else %} + + {% endif %} + {% if feed.error_count >= application.config['DEFAULT_MAX_ERROR'] %} + + {% endif %} + {% if feed.icon_url %} {% endif %}{{ feed.title }}{{ feed.site_link }}( {{ unread_article_count.get(feed.id, 0) }} ) {{ article_count.get(feed.id, 0) }} + + + + +
+
+ +{% endif %} diff --git a/newspipe/templates/feed_list_per_categories.html b/newspipe/templates/feed_list_per_categories.html new file mode 100644 index 00000000..34d10ddd --- /dev/null +++ b/newspipe/templates/feed_list_per_categories.html @@ -0,0 +1,52 @@ +
+
+
+
+ + + +
+
+
+
+ +
+ +
+ + + + + + + + + + {% for feed in feeds %} + + + + + + {% endfor %} + +
#{{ _('Title') }}{{ _('Site') }}
{{ loop.index }}{% if feed.icon_url %} {% endif %} {{ feed.title }}{{ feed.site_link }}
+
+ diff --git a/newspipe/templates/feed_list_simple.html b/newspipe/templates/feed_list_simple.html new file mode 100644 index 00000000..5f692a53 --- /dev/null +++ b/newspipe/templates/feed_list_simple.html @@ -0,0 +1,35 @@ +{% if feeds | length != 0 %} +
+ + + + + + + + + + {% for feed in feeds %} + + + + + + {% endfor %} + +
#{{ _('Title') }}{{ _('Site') }}
{{ loop.index }}{% if feed.icon_url %} {% endif %} {{ feed.title }}{{ feed.site_link }}
+
+ +{% endif %} diff --git a/newspipe/templates/feeds.html b/newspipe/templates/feeds.html new file mode 100644 index 00000000..805e1b74 --- /dev/null +++ b/newspipe/templates/feeds.html @@ -0,0 +1,7 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ _('You are subscribed to %(feed_count)d feeds.', feed_count=feeds.count()) }} {{ _('Add') }} {{ _('a feed') }}.

+ {% include "feed_list.html" %} +
+{% endblock %} diff --git a/newspipe/templates/history.html b/newspipe/templates/history.html new file mode 100644 index 00000000..ba567106 --- /dev/null +++ b/newspipe/templates/history.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ _('History') }}

+ {% if month != None %} +

{{ year }}

+

{{ month | month_name }}

+ {% elif year != None %} +

 {{ _('all years') }}

+

{{ year }}

+ {% endif %} +
    + {% for article in articles_counter | sort(reverse = True) %} + {% if year == None %} +
  • {{ article }} : {{ articles_counter[article] }} articles
    • + {% elif month == None %} +
  • {{ article | month_name }} : {{ articles_counter[article] }} articles
    • + {% else %} + {% for article in articles %} +
  • {{ article.date | datetime }} - {{ article.title | safe }}
    • + {% endfor %} + {% endif %} + {% endfor %} +
+
+{% endblock %} diff --git a/newspipe/templates/home.html b/newspipe/templates/home.html new file mode 100644 index 00000000..8da8dbe1 --- /dev/null +++ b/newspipe/templates/home.html @@ -0,0 +1,135 @@ +{% extends "layout.html" %} +{% block content %} + +
+{% if feeds|count == 0 %} +
+

{{ _("You don't have any feeds.") }}

+

{{ _('Add some') }}, {{ _('or') }} {{ _('upload an OPML file.') }}

+
+{% else %} +
+ +
+ + {% if articles | count != 0%} +
+ + + + + + + + + + + {% for article in articles %} + + + + + + + {% endfor %} + +
{{ _('Feed') }}{{ _('Article') }}{{ _('Date') }}
+ + {% if article.like %} + + {% else %} + + {% endif %} + {% if article.readed %} + + {% else %} + + {% if filter_ == 'all' %}{% endif %} + {% endif %} + {{ article.source.title | safe }} + {{ article.title | safe }} + {{ article.date | datetime }}
+
+ {% endif %} +
+
+{% endif %} +
+{% endblock %} diff --git a/newspipe/templates/inactives.html b/newspipe/templates/inactives.html new file mode 100644 index 00000000..e89a5fe1 --- /dev/null +++ b/newspipe/templates/inactives.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+

{{ _('Days of inactivity') }}:

+ +
+
+ {% if inactives != [] %} + + {% else %} +

{{ _('No inactive feeds.') }}

+ {% endif %} +

+
+{% endblock %} diff --git a/newspipe/templates/layout.html b/newspipe/templates/layout.html new file mode 100644 index 00000000..b40b540c --- /dev/null +++ b/newspipe/templates/layout.html @@ -0,0 +1,140 @@ + + + + {% block head %} + + + + + Newspipe{% if head_titles %} - {{ ' - '.join(head_titles) }}{% endif %} + + + + + + + + + + + + {% endblock %} + + + {% block menu %} + + {% endblock %} +
+ +
+ {% block messages %} + {% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + {% for category, message in messages %} +
+ + {{ message }} +
+ {% endfor %} + {% endif %} + {% endwith %} + {% endblock %} +
+ + {% block content %}{% endblock %} + + + + + + diff --git a/newspipe/templates/login.html b/newspipe/templates/login.html new file mode 100644 index 00000000..1253e2d3 --- /dev/null +++ b/newspipe/templates/login.html @@ -0,0 +1,32 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+
+
+

{{ _('Log In') }}

+
+ {{ form.hidden_tag() }} +
+ {{ form.nickmane(class_="form-control", placeholder=_('Your nickname')) }} +
+ {% for message in form.nickmane.errors %} + + {% endfor %} +
+ {{ form.password(class_="form-control", placeholder=_('Your Password')) }} +
+ {% for message in form.password.errors %} + + {% endfor %} + {{ form.submit(class_="btn btn-primary") }} +
+
+
+
+ {{ _('Sign up') }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/management.html b/newspipe/templates/management.html new file mode 100644 index 00000000..22ebde8a --- /dev/null +++ b/newspipe/templates/management.html @@ -0,0 +1,99 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+
+
+

{{ _('Your subscriptions') }}

+

{{ _('You are subscribed to') }} {{ nb_feeds }} {{ _('feeds') }}. {{ _('Add') }} {{ _('a feed') }}.

+

{{ nb_articles }} {{ _('articles are stored in the database with') }} {{ nb_unread_articles }} {{ _('unread articles') }}.

+

{{ _('You have') }} {{ nb_categories }} {{ _('categories') }}.

+ {{ _('Delete articles older than 10 weeks') }} +
+
+

{{ _('Your bookmarks') }}

+

{{ _('You have') }} {{ nb_bookmarks }} {{ _('bookmarks') }}.

+ {{ _('Delete all bookmarks') }} +
+
+
+
+
+
+
+
+
+

{{ _('Your data') }}

+
+
+
+
+
+

{{ _('Articles') }}

+
+
+
+
+
{{ _('Import') }}
+

{{ _('Import a Newspipe account (*.json).') }}

+
+ + +
+
+
+
{{ _('Export') }}
+

{{ _('Export your newspipe account to JSON.') }}

+ {{ _('Export') }} +
+
+
+
+
+

{{ _('Feeds') }}

+
+
+
+
+
{{ _('Import') }}
+

{{ _('Import feeds from OPML (*.xml or *.opml).') }}

+
+ + +
+
+
+
{{ _('Export') }}
+

{{ _('Export feeds to OPML.') }}

+
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+ +
+
+
+

{{ _('Bookmarks') }}

+ {{ _('Export your bookmarks to JSON') }} +
+
+
+
+
+{% endblock %} diff --git a/newspipe/templates/opml.xml b/newspipe/templates/opml.xml new file mode 100644 index 00000000..7159e279 --- /dev/null +++ b/newspipe/templates/opml.xml @@ -0,0 +1,13 @@ + + + + + Feeds of {{ user.nickname }} + {{ now | datetime }} + {{ now | datetime }} + {{ user.nickname }} + + + {% for feed in feeds %} + {% endfor %} + diff --git a/newspipe/templates/popular.html b/newspipe/templates/popular.html new file mode 100644 index 00000000..a80cee57 --- /dev/null +++ b/newspipe/templates/popular.html @@ -0,0 +1,27 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+

{{ _('Popular feeds') }}

+ all ‧ last year ‧ last month +
+
+
+
+ +
+
+
+{% endblock %} diff --git a/newspipe/templates/profile.html b/newspipe/templates/profile.html new file mode 100644 index 00000000..edbae368 --- /dev/null +++ b/newspipe/templates/profile.html @@ -0,0 +1,68 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ _('Your Profile') }}

+
+
+

{{ _('Member since') }}: {{ user.date_created | datetime }}.

+

{{ _('Last seen') }}: {{ user.last_seen | datetime }}.

+
+
+
+
+
+
+

Edit your profile

+
+
+
+
+ {{ form.hidden_tag() }} + +
+ {{ form.nickname.label }} + {{ form.nickname(class_="form-control") }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} + + {{ form.password.label }} + {{ form.password(class_="form-control") }} {% for error in form.password.errors %} {{ error }}
{% endfor %} + + {{ form.password_conf.label }} + {{ form.password_conf(class_="form-control") }} {% for error in form.password_conf.errors %} {{ error }}
{% endfor %} +
+ +
+ {{ form.bio.label }} + {{ form.bio(class_="form-control") }} {% for error in form.bio.errors %} {{ error }}
{% endfor %} + + {{ form.webpage.label }} + {{ form.webpage(class_="form-control") }} {% for error in form.webpage.errors %} {{ error }}
{% endfor %} + + {{ form.twitter.label }} + {{ form.twitter(class_="form-control") }} {% for error in form.twitter.errors %} {{ error }}
{% endfor %} + + {{ form.is_public_profile.label }} + {{ form.is_public_profile(class_="form-control") }} {% for error in form.is_public_profile.errors %} {{ error }}
{% endfor %} +

{{ _('Your profile will be available here.', url=url_for('user.profile_public', nickname=user.nickname) ) }}

+ + {{ form.automatic_crawling.label }} + {{ form.automatic_crawling(class_="form-control") }} {% for error in form.automatic_crawling.errors %} {{ error }}
{% endfor %} +

{{ _('Uncheck if you are using your own crawler.') }}

+
+
+
+
+
+ {{ form.submit(class_="btn btn-primary") }} +
+
+
+
+
+
+
+
+ {{ _('Delete your account') }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/profile_public.html b/newspipe/templates/profile_public.html new file mode 100644 index 00000000..9ba3578c --- /dev/null +++ b/newspipe/templates/profile_public.html @@ -0,0 +1,45 @@ +{% extends "layout.html" %} +{% block content %} +
+

{{ user.nickname }} / stream

+
+
+

+   + {{ _('Member since') }}: {{ user.date_created | datetime }} +

+

+   + {{ _('Last seen') }}: {{ user.last_seen | datetime }} +

+ {% if user.webpage %} +

+   + {{ _('Webpage') }}: {{ user.webpage | safe }} +

+ {% endif %} + {% if user.twitter %} +

+   + {{ _('Twitter') }}: {{ user.twitter | safe }} +

+ {% endif %} +
+
+
+
+ {% if user.bio %} +

{{ user.bio }}

+ {% endif %} +
+
+
+ +

{{ _('Feeds') }}

+
+
+ {% include "feed_list_per_categories.html" %} +
+
+
+{% endblock %} diff --git a/newspipe/templates/signup.html b/newspipe/templates/signup.html new file mode 100644 index 00000000..8d34b3bf --- /dev/null +++ b/newspipe/templates/signup.html @@ -0,0 +1,24 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+ {{ form.hidden_tag() }} +
+ {{ form.nickname(class_="form-control", placeholder=_('Your nickname')) }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} +

{{ _('Letters, numbers, dots and underscores only.') }}

+
+
+ {{ form.email(class_="form-control", placeholder=_('Your email')) }} {% for error in form.email.errors %} {{ error }}
{% endfor %} +

{{ _("Only for account activation. Your email won't be stored.") }}

+
+
+ {{ form.password(class_="form-control", placeholder=_('Your password')) }} {% for error in form.password.errors %} {{ error }}
{% endfor %} +

{{ _('Minimum 6 characters.') }}

+
+
+ {{ form.submit(class_="btn btn-default") }} +
+
+
+{% endblock %} diff --git a/newspipe/templates/user_stream.html b/newspipe/templates/user_stream.html new file mode 100644 index 00000000..b05376a8 --- /dev/null +++ b/newspipe/templates/user_stream.html @@ -0,0 +1,70 @@ +{% extends "layout.html" %} +{% block content %} +
+
+
+
+
+ + + +
+
+
+
+ +

+ + {% if category %} +
+
+

Articles from the category {{ category.name }}

+
+
+ {% endif %} + +
+
+ {{ pagination.info }} +
+
+ +
+
+ {{ pagination.links }} +
+
+ +
+ + + + + + + + + + {% for article in articles %} + + + + + + {% endfor %} + +
#{{ _('Title') }}{{ _('Published at') }}
{{ loop.index }}{{ article.title }}{{ article.date }}
+
+ +
+
+ {{ pagination.links }} +
+
+
+{% endblock %} diff --git a/newspipe/web/controllers/__init__.py b/newspipe/web/controllers/__init__.py deleted file mode 100644 index 9b193cc5..00000000 --- a/newspipe/web/controllers/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from .feed import FeedController -from .category import CategoryController -from .article import ArticleController -from .user import UserController -from .icon import IconController -from .bookmark import BookmarkController -from .tag import BookmarkTagController - - -__all__ = [ - "FeedController", - "CategoryController", - "ArticleController", - "UserController", - "IconController", - "BookmarkController", - "BookmarkTagController", -] diff --git a/newspipe/web/controllers/abstract.py b/newspipe/web/controllers/abstract.py deleted file mode 100644 index 9d9e84f2..00000000 --- a/newspipe/web/controllers/abstract.py +++ /dev/null @@ -1,177 +0,0 @@ -import logging -import dateutil.parser -from bootstrap import db -from datetime import datetime -from collections import defaultdict -from sqlalchemy import or_, func -from werkzeug.exceptions import Forbidden, NotFound - -logger = logging.getLogger(__name__) - - -class AbstractController: - _db_cls = None # reference to the database class - _user_id_key = "user_id" - - def __init__(self, user_id=None, ignore_context=False): - """User id is a right management mechanism that should be used to - filter objects in database on their denormalized "user_id" field - (or "id" field for users). - Should no user_id be provided, the Controller won't apply any filter - allowing for a kind of "super user" mode. - """ - try: - self.user_id = int(user_id) - except TypeError: - self.user_id = user_id - - def _to_filters(self, **filters): - """ - Will translate filters to sqlalchemy filter. - This method will also apply user_id restriction if available. - - each parameters of the function is treated as an equality unless the - name of the parameter ends with either "__gt", "__lt", "__ge", "__le", - "__ne", "__in" ir "__like". - """ - db_filters = set() - for key, value in filters.items(): - if key == "__or__": - db_filters.add(or_(*self._to_filters(**value))) - elif key.endswith("__gt"): - db_filters.add(getattr(self._db_cls, key[:-4]) > value) - elif key.endswith("__lt"): - db_filters.add(getattr(self._db_cls, key[:-4]) < value) - elif key.endswith("__ge"): - db_filters.add(getattr(self._db_cls, key[:-4]) >= value) - elif key.endswith("__le"): - db_filters.add(getattr(self._db_cls, key[:-4]) <= value) - elif key.endswith("__ne"): - db_filters.add(getattr(self._db_cls, key[:-4]) != value) - elif key.endswith("__in"): - db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) - elif key.endswith("__contains"): - db_filters.add(getattr(self._db_cls, key[:-10]).contains(value)) - elif key.endswith("__like"): - db_filters.add(getattr(self._db_cls, key[:-6]).like(value)) - elif key.endswith("__ilike"): - db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value)) - else: - db_filters.add(getattr(self._db_cls, key) == value) - return db_filters - - def _get(self, **filters): - """ Will add the current user id if that one is not none (in which case - the decision has been made in the code that the query shouldn't be user - dependent) and the user is not an admin and the filters doesn't already - contains a filter for that user. - """ - if ( - self._user_id_key is not None - and self.user_id - and filters.get(self._user_id_key) != self.user_id - ): - filters[self._user_id_key] = self.user_id - return self._db_cls.query.filter(*self._to_filters(**filters)) - - def get(self, **filters): - """Will return one single objects corresponding to filters""" - obj = self._get(**filters).first() - - if obj and not self._has_right_on(obj): - raise Forbidden( - { - "message": "No authorized to access %r (%r)" - % (self._db_cls.__class__.__name__, filters) - } - ) - if not obj: - raise NotFound( - {"message": "No %r (%r)" % (self._db_cls.__class__.__name__, filters)} - ) - return obj - - def create(self, **attrs): - assert attrs, "attributes to update must not be empty" - if self._user_id_key is not None and self._user_id_key not in attrs: - attrs[self._user_id_key] = self.user_id - assert ( - self._user_id_key is None - or self._user_id_key in attrs - or self.user_id is None - ), "You must provide user_id one way or another" - - obj = self._db_cls(**attrs) - db.session.add(obj) - db.session.flush() - db.session.commit() - return obj - - def read(self, **filters): - return self._get(**filters) - - def update(self, filters, attrs, return_objs=False, commit=True): - assert attrs, "attributes to update must not be empty" - result = self._get(**filters).update(attrs, synchronize_session=False) - if commit: - db.session.flush() - db.session.commit() - if return_objs: - return self._get(**filters) - return result - - def delete(self, obj_id): - obj = self.get(id=obj_id) - db.session.delete(obj) - try: - db.session.commit() - except Exception as e: - db.session.rollback() - return obj - - def _has_right_on(self, obj): - # user_id == None is like being admin - if self._user_id_key is None: - return True - return ( - self.user_id is None - or getattr(obj, self._user_id_key, None) == self.user_id - ) - - def _count_by(self, elem_to_group_by, filters): - if self.user_id: - filters["user_id"] = self.user_id - return dict( - db.session.query(elem_to_group_by, func.count("id")) - .filter(*self._to_filters(**filters)) - .group_by(elem_to_group_by) - .all() - ) - - @classmethod - def _get_attrs_desc(cls, role, right=None): - result = defaultdict(dict) - if role == "admin": - columns = cls._db_cls.__table__.columns.keys() - else: - assert role in {"base", "api"}, "unknown role %r" % role - assert right in {"read", "write"}, ( - "right must be 'read' or 'write' with role %r" % role - ) - columns = getattr(cls._db_cls, "fields_%s_%s" % (role, right))() - for column in columns: - result[column] = {} - db_col = getattr(cls._db_cls, column).property.columns[0] - try: - result[column]["type"] = db_col.type.python_type - except NotImplementedError: - if db_col.default: - result[column]["type"] = db_col.default.arg.__class__ - if column not in result: - continue - if issubclass(result[column]["type"], datetime): - result[column]["default"] = datetime.utcnow() - result[column]["type"] = lambda x: dateutil.parser.parse(x) - elif db_col.default: - result[column]["default"] = db_col.default.arg - return result diff --git a/newspipe/web/controllers/article.py b/newspipe/web/controllers/article.py deleted file mode 100644 index d5efcb74..00000000 --- a/newspipe/web/controllers/article.py +++ /dev/null @@ -1,107 +0,0 @@ -import re -import logging -import sqlalchemy -from sqlalchemy import func -from collections import Counter - -from bootstrap import db -from .abstract import AbstractController -from lib.article_utils import process_filters -from web.controllers import CategoryController, FeedController -from web.models import Article - -logger = logging.getLogger(__name__) - - -class ArticleController(AbstractController): - _db_cls = Article - - def challenge(self, ids): - """Will return each id that wasn't found in the database.""" - for id_ in ids: - if self.read(**id_).first(): - continue - yield id_ - - def count_by_category(self, **filters): - return self._count_by(Article.category_id, filters) - - def count_by_feed(self, **filters): - return self._count_by(Article.feed_id, filters) - - def count_by_user_id(self, **filters): - return dict( - db.session.query(Article.user_id, func.count(Article.id)) - .filter(*self._to_filters(**filters)) - .group_by(Article.user_id) - .all() - ) - - def create(self, **attrs): - # handling special denorm for article rights - assert "feed_id" in attrs, "must provide feed_id when creating article" - feed = FeedController(attrs.get("user_id", self.user_id)).get( - id=attrs["feed_id"] - ) - if "user_id" in attrs: - assert feed.user_id == attrs["user_id"] or self.user_id is None, ( - "no right on feed %r" % feed.id - ) - attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id - - skipped, read, liked = process_filters(feed.filters, attrs) - if skipped: - return None - article = super().create(**attrs) - return article - - def update(self, filters, attrs): - user_id = attrs.get("user_id", self.user_id) - if "feed_id" in attrs: - feed = FeedController().get(id=attrs["feed_id"]) - assert feed.user_id == user_id, "no right on feed %r" % feed.id - attrs["category_id"] = feed.category_id - if attrs.get("category_id"): - cat = CategoryController().get(id=attrs["category_id"]) - assert self.user_id is None or cat.user_id == user_id, ( - "no right on cat %r" % cat.id - ) - return super().update(filters, attrs) - - def get_history(self, year=None, month=None): - """ - Sort articles by year and month. - """ - articles_counter = Counter() - articles = self.read() - if year is not None: - articles = articles.filter(sqlalchemy.extract("year", Article.date) == year) - if month is not None: - articles = articles.filter( - sqlalchemy.extract("month", Article.date) == month - ) - for article in articles.all(): - if year is not None: - articles_counter[article.date.month] += 1 - else: - articles_counter[article.date.year] += 1 - return articles_counter, articles - - def read_light(self, **filters): - return ( - super() - .read(**filters) - .with_entities( - Article.id, - Article.title, - Article.readed, - Article.like, - Article.feed_id, - Article.date, - Article.category_id, - ) - .order_by(Article.date.desc()) - ) - - def read_ordered(self, **filters): - return super().read(**filters).order_by(Article.date.desc()) diff --git a/newspipe/web/controllers/bookmark.py b/newspipe/web/controllers/bookmark.py deleted file mode 100644 index d1c1260c..00000000 --- a/newspipe/web/controllers/bookmark.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging -import itertools -from datetime import datetime, timedelta - -from bootstrap import db -from web.models import Bookmark -from .abstract import AbstractController -from .tag import BookmarkTagController - -logger = logging.getLogger(__name__) - - -class BookmarkController(AbstractController): - _db_cls = Bookmark - - def count_by_href(self, **filters): - return self._count_by(Bookmark.href, filters) - - def update(self, filters, attrs): - BookmarkTagController(self.user_id).read( - **{"bookmark_id": filters["id"]} - ).delete() - - for tag in attrs["tags"]: - BookmarkTagController(self.user_id).create( - **{ - "text": tag.text, - "id": tag.id, - "bookmark_id": tag.bookmark_id, - "user_id": tag.user_id, - } - ) - - del attrs["tags"] - return super().update(filters, attrs) diff --git a/newspipe/web/controllers/category.py b/newspipe/web/controllers/category.py deleted file mode 100644 index ec54f5a3..00000000 --- a/newspipe/web/controllers/category.py +++ /dev/null @@ -1,13 +0,0 @@ -from .abstract import AbstractController -from web.models import Category -from .feed import FeedController - - -class CategoryController(AbstractController): - _db_cls = Category - - def delete(self, obj_id): - FeedController(self.user_id).update( - {"category_id": obj_id}, {"category_id": None} - ) - return super().delete(obj_id) diff --git a/newspipe/web/controllers/feed.py b/newspipe/web/controllers/feed.py deleted file mode 100644 index 19ba463f..00000000 --- a/newspipe/web/controllers/feed.py +++ /dev/null @@ -1,105 +0,0 @@ -import logging -import itertools -from datetime import datetime, timedelta - -import conf -from .abstract import AbstractController -from .icon import IconController -from web.models import User, Feed -from lib.utils import clear_string - -logger = logging.getLogger(__name__) -DEFAULT_LIMIT = 5 -DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR - - -class FeedController(AbstractController): - _db_cls = Feed - - def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): - return [ - feed - for feed in self.read( - error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last - ) - .join(User) - .filter(User.is_active == True) - .order_by("last_retrieved") - .limit(limit) - ] - - def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): - now = datetime.now() - max_last = now - timedelta(minutes=60) - feeds = self.list_late(max_last, max_error, limit) - if feeds: - self.update( - {"id__in": [feed.id for feed in feeds]}, {"last_retrieved": now} - ) - return feeds - - def get_duplicates(self, feed_id): - """ - Compare a list of documents by pair. - Pairs of duplicates are sorted by "retrieved date". - """ - feed = self.get(id=feed_id) - duplicates = [] - for pair in itertools.combinations(feed.articles[:1000], 2): - date1, date2 = pair[0].date, pair[1].date - if clear_string(pair[0].title) == clear_string(pair[1].title) and ( - date1 - date2 - ) < timedelta(days=1): - if pair[0].retrieved_date < pair[1].retrieved_date: - duplicates.append((pair[0], pair[1])) - else: - duplicates.append((pair[1], pair[0])) - return feed, duplicates - - def get_inactives(self, nb_days): - today = datetime.now() - inactives = [] - for feed in self.read(): - try: - last_post = feed.articles[0].date - except IndexError: - continue - except Exception as e: - logger.exception(e) - continue - elapsed = today - last_post - if elapsed > timedelta(days=nb_days): - inactives.append((feed, elapsed)) - inactives.sort(key=lambda tup: tup[1], reverse=True) - return inactives - - def count_by_category(self, **filters): - return self._count_by(Feed.category_id, filters) - - def count_by_link(self, **filters): - return self._count_by(Feed.link, filters) - - def _ensure_icon(self, attrs): - if not attrs.get("icon_url"): - return - icon_contr = IconController() - if not icon_contr.read(url=attrs["icon_url"]).count(): - icon_contr.create(**{"url": attrs["icon_url"]}) - - def create(self, **attrs): - self._ensure_icon(attrs) - return super().create(**attrs) - - def update(self, filters, attrs): - from .article import ArticleController - - self._ensure_icon(attrs) - if "category_id" in attrs and attrs["category_id"] == 0: - del attrs["category_id"] - elif "category_id" in attrs: - art_contr = ArticleController(self.user_id) - for feed in self.read(**filters): - art_contr.update( - {"feed_id": feed.id}, {"category_id": attrs["category_id"]} - ) - return super().update(filters, attrs) diff --git a/newspipe/web/controllers/icon.py b/newspipe/web/controllers/icon.py deleted file mode 100644 index de86b52f..00000000 --- a/newspipe/web/controllers/icon.py +++ /dev/null @@ -1,27 +0,0 @@ -import base64 -import requests -from web.models import Icon -from .abstract import AbstractController - - -class IconController(AbstractController): - _db_cls = Icon - _user_id_key = None - - def _build_from_url(self, attrs): - if "url" in attrs and "content" not in attrs: - resp = requests.get(attrs["url"], verify=False) - attrs.update( - { - "url": resp.url, - "mimetype": resp.headers.get("content-type", None), - "content": base64.b64encode(resp.content).decode("utf8"), - } - ) - return attrs - - def create(self, **attrs): - return super().create(**self._build_from_url(attrs)) - - def update(self, filters, attrs): - return super().update(filters, self._build_from_url(attrs)) diff --git a/newspipe/web/controllers/tag.py b/newspipe/web/controllers/tag.py deleted file mode 100644 index 35fd5613..00000000 --- a/newspipe/web/controllers/tag.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -import itertools -from datetime import datetime, timedelta - -from bootstrap import db -from .abstract import AbstractController -from web.models.tag import BookmarkTag - -logger = logging.getLogger(__name__) - - -class BookmarkTagController(AbstractController): - _db_cls = BookmarkTag - - def count_by_href(self, **filters): - return self._count_by(BookmarkTag.text, filters) - - def create(self, **attrs): - return super().create(**attrs) - - def update(self, filters, attrs): - return super().update(filters, attrs) diff --git a/newspipe/web/controllers/user.py b/newspipe/web/controllers/user.py deleted file mode 100644 index 71eb7d08..00000000 --- a/newspipe/web/controllers/user.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -from werkzeug.security import generate_password_hash, check_password_hash -from .abstract import AbstractController -from web.models import User - -logger = logging.getLogger(__name__) - - -class UserController(AbstractController): - _db_cls = User - _user_id_key = "id" - - def _handle_password(self, attrs): - if attrs.get("password"): - attrs["pwdhash"] = generate_password_hash(attrs.pop("password")) - elif "password" in attrs: - del attrs["password"] - - def check_password(self, user, password): - return check_password_hash(user.pwdhash, password) - - def create(self, **attrs): - self._handle_password(attrs) - return super().create(**attrs) - - def update(self, filters, attrs): - self._handle_password(attrs) - return super().update(filters, attrs) diff --git a/newspipe/web/forms.py b/newspipe/web/forms.py index e1921210..540b2723 100644 --- a/newspipe/web/forms.py +++ b/newspipe/web/forms.py @@ -43,9 +43,9 @@ from wtforms import ( ) from wtforms.fields.html5 import EmailField, URLField -from lib import misc_utils -from web.controllers import UserController -from web.models import User +from newspipe.lib import misc_utils +from newspipe.controllers import UserController +from newspipe.models import User class SignupForm(FlaskForm): diff --git a/newspipe/web/lib/user_utils.py b/newspipe/web/lib/user_utils.py index 84b1c75c..95b436a9 100644 --- a/newspipe/web/lib/user_utils.py +++ b/newspipe/web/lib/user_utils.py @@ -1,6 +1,6 @@ from itsdangerous import URLSafeTimedSerializer -import conf -from bootstrap import application + +from newspipe.bootstrap import application def generate_confirmation_token(nickname): @@ -14,7 +14,7 @@ def confirm_token(token): nickname = serializer.loads( token, salt=application.config["SECURITY_PASSWORD_SALT"], - max_age=conf.TOKEN_VALIDITY_PERIOD, + max_age=application.config['TOKEN_VALIDITY_PERIOD'], ) except: return False diff --git a/newspipe/web/lib/view_utils.py b/newspipe/web/lib/view_utils.py index 218ebb4c..c6e722d3 100644 --- a/newspipe/web/lib/view_utils.py +++ b/newspipe/web/lib/view_utils.py @@ -1,6 +1,6 @@ from functools import wraps from flask import request, Response, make_response -from lib.utils import to_hash +from newspipe.lib.utils import to_hash def etag_match(func): diff --git a/newspipe/web/models/__init__.py b/newspipe/web/models/__init__.py deleted file mode 100644 index a58a7ad5..00000000 --- a/newspipe/web/models/__init__.py +++ /dev/null @@ -1,98 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# Newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -from .feed import Feed -from .role import Role -from .user import User -from .article import Article -from .icon import Icon -from .category import Category -from .tag import BookmarkTag -from .tag import ArticleTag -from .bookmark import Bookmark - -__all__ = [ - "Feed", - "Role", - "User", - "Article", - "Icon", - "Category", - "Bookmark", - "ArticleTag", - "BookmarkTag", -] - -import os - -from sqlalchemy.engine import reflection -from sqlalchemy.schema import ( - MetaData, - Table, - DropTable, - ForeignKeyConstraint, - DropConstraint, -) - - -def db_empty(db): - "Will drop every datas stocked in db." - # From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything - conn = db.engine.connect() - - # the transaction only applies if the DB supports - # transactional DDL, i.e. Postgresql, MS SQL Server - trans = conn.begin() - - inspector = reflection.Inspector.from_engine(db.engine) - - # gather all data first before dropping anything. - # some DBs lock after things have been dropped in - # a transaction. - metadata = MetaData() - - tbs = [] - all_fks = [] - - for table_name in inspector.get_table_names(): - fks = [] - for fk in inspector.get_foreign_keys(table_name): - if not fk["name"]: - continue - fks.append(ForeignKeyConstraint((), (), name=fk["name"])) - t = Table(table_name, metadata, *fks) - tbs.append(t) - all_fks.extend(fks) - - for fkc in all_fks: - conn.execute(DropConstraint(fkc)) - - for table in tbs: - conn.execute(DropTable(table)) - - trans.commit() diff --git a/newspipe/web/models/article.py b/newspipe/web/models/article.py deleted file mode 100644 index 4e623ba2..00000000 --- a/newspipe/web/models/article.py +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# Newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.5 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2016/10/04 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -from bootstrap import db -from datetime import datetime -from sqlalchemy import Index -from sqlalchemy.ext.associationproxy import association_proxy - -from web.models.right_mixin import RightMixin - - -class Article(db.Model, RightMixin): - "Represent an article from a feed." - id = db.Column(db.Integer(), primary_key=True) - entry_id = db.Column(db.String(), nullable=False) - link = db.Column(db.String()) - title = db.Column(db.String()) - content = db.Column(db.String()) - readed = db.Column(db.Boolean(), default=False) - like = db.Column(db.Boolean(), default=False) - date = db.Column(db.DateTime(), default=datetime.utcnow) - updated_date = db.Column(db.DateTime(), default=datetime.utcnow) - retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow) - - # foreign keys - user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) - feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id")) - category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) - - # relationships - tag_objs = db.relationship( - "ArticleTag", - back_populates="article", - cascade="all,delete-orphan", - lazy=False, - foreign_keys="[ArticleTag.article_id]", - ) - tags = association_proxy("tag_objs", "text") - - # indexes - # __table_args__ = ( - # Index('user_id'), - # Index('user_id', 'category_id'), - # Index('user_id', 'feed_id'), - # Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id) - # ) - - # api whitelists - @staticmethod - def _fields_base_write(): - return {"readed", "like", "feed_id", "category_id"} - - @staticmethod - def _fields_base_read(): - return { - "id", - "entry_id", - "link", - "title", - "content", - "date", - "retrieved_date", - "user_id", - "tags", - } - - @staticmethod - def _fields_api_write(): - return {"tags"} - - def __repr__(self): - return ( - "" - % (self.id, self.entry_id, self.title, self.date, self.retrieved_date) - ) diff --git a/newspipe/web/models/bookmark.py b/newspipe/web/models/bookmark.py deleted file mode 100644 index 11b2db53..00000000 --- a/newspipe/web/models/bookmark.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# Newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.1 $" -__date__ = "$Date: 2016/12/07 $" -__revision__ = "$Date: 2016/12/07 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -from bootstrap import db -from datetime import datetime -from sqlalchemy import desc -from sqlalchemy.orm import validates -from sqlalchemy.ext.associationproxy import association_proxy - -from web.models.tag import BookmarkTag -from web.models.right_mixin import RightMixin - - -class Bookmark(db.Model, RightMixin): - """ - Represent a bookmark. - """ - - id = db.Column(db.Integer(), primary_key=True) - href = db.Column(db.String(), default="") - title = db.Column(db.String(), default="") - description = db.Column(db.String(), default="") - shared = db.Column(db.Boolean(), default=False) - to_read = db.Column(db.Boolean(), default=False) - time = db.Column(db.DateTime(), default=datetime.utcnow) - user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) - - # relationships - tags = db.relationship( - BookmarkTag, - backref="of_bookmark", - lazy="dynamic", - cascade="all,delete-orphan", - order_by=desc(BookmarkTag.text), - ) - tags_proxy = association_proxy("tags", "text") - - @validates("description") - def validates_title(self, key, value): - return str(value).strip() - - @validates("extended") - def validates_description(self, key, value): - return str(value).strip() - - def __repr__(self): - return "" % (self.href) diff --git a/newspipe/web/models/category.py b/newspipe/web/models/category.py deleted file mode 100644 index bb47ce45..00000000 --- a/newspipe/web/models/category.py +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from bootstrap import db -from sqlalchemy import Index -from web.models.right_mixin import RightMixin - - -class Category(db.Model, RightMixin): - id = db.Column(db.Integer(), primary_key=True) - name = db.Column(db.String()) - - # relationships - user_id = db.Column(db.Integer, db.ForeignKey("user.id")) - feeds = db.relationship("Feed", cascade="all,delete-orphan") - articles = db.relationship("Article", cascade="all,delete-orphan") - - # index - idx_category_uid = Index("user_id") - - # api whitelists - @staticmethod - def _fields_base_read(): - return {"id", "user_id"} - - @staticmethod - def _fields_base_write(): - return {"name"} diff --git a/newspipe/web/models/feed.py b/newspipe/web/models/feed.py deleted file mode 100644 index 70a5d521..00000000 --- a/newspipe/web/models/feed.py +++ /dev/null @@ -1,105 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -from bootstrap import db -from datetime import datetime -from sqlalchemy import desc, Index -from sqlalchemy.orm import validates -from web.models.right_mixin import RightMixin -from web.models.article import Article - - -class Feed(db.Model, RightMixin): - """ - Represent a feed. - """ - - id = db.Column(db.Integer(), primary_key=True) - title = db.Column(db.String(), default="") - description = db.Column(db.String(), default="FR") - link = db.Column(db.String(), nullable=False) - site_link = db.Column(db.String(), default="") - enabled = db.Column(db.Boolean(), default=True) - created_date = db.Column(db.DateTime(), default=datetime.utcnow) - filters = db.Column(db.PickleType, default=[]) - private = db.Column(db.Boolean(), default=False) - - # cache handling - etag = db.Column(db.String(), default="") - last_modified = db.Column(db.String(), default="") - last_retrieved = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) - - # error logging - last_error = db.Column(db.String(), default="") - error_count = db.Column(db.Integer(), default=0) - - # relationship - icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None) - user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) - category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) - articles = db.relationship( - Article, - backref="source", - lazy="dynamic", - cascade="all,delete-orphan", - order_by=desc(Article.date), - ) - - # index - idx_feed_uid_cid = Index("user_id", "category_id") - idx_feed_uid = Index("user_id") - - # api whitelists - @staticmethod - def _fields_base_write(): - return { - "title", - "description", - "link", - "site_link", - "enabled", - "filters", - "last_error", - "error_count", - "category_id", - } - - @staticmethod - def _fields_base_read(): - return {"id", "user_id", "icon_url", "last_retrieved"} - - @validates("title") - def validates_title(self, key, value): - return str(value).strip() - - @validates("description") - def validates_description(self, key, value): - return str(value).strip() - - def __repr__(self): - return "" % (self.title) diff --git a/newspipe/web/models/icon.py b/newspipe/web/models/icon.py deleted file mode 100644 index adc9cf69..00000000 --- a/newspipe/web/models/icon.py +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from bootstrap import db - - -class Icon(db.Model): - url = db.Column(db.String(), primary_key=True) - content = db.Column(db.String(), default=None) - mimetype = db.Column(db.String(), default="application/image") diff --git a/newspipe/web/models/right_mixin.py b/newspipe/web/models/right_mixin.py deleted file mode 100644 index 670beafa..00000000 --- a/newspipe/web/models/right_mixin.py +++ /dev/null @@ -1,64 +0,0 @@ -from sqlalchemy.ext.associationproxy import _AssociationList - - -class RightMixin: - @staticmethod - def _fields_base_write(): - return set() - - @staticmethod - def _fields_base_read(): - return set(["id"]) - - @staticmethod - def _fields_api_write(): - return set([]) - - @staticmethod - def _fields_api_read(): - return set(["id"]) - - @classmethod - def fields_base_write(cls): - return cls._fields_base_write() - - @classmethod - def fields_base_read(cls): - return cls._fields_base_write().union(cls._fields_base_read()) - - @classmethod - def fields_api_write(cls): - return cls.fields_base_write().union(cls._fields_api_write()) - - @classmethod - def fields_api_read(cls): - return cls.fields_base_read().union(cls._fields_api_read()) - - def __getitem__(self, key): - if not hasattr(self, "__dump__"): - self.__dump__ = {} - return self.__dump__.get(key) - - def __setitem__(self, key, value): - if not hasattr(self, "__dump__"): - self.__dump__ = {} - self.__dump__[key] = value - - def dump(self, role="admin"): - if role == "admin": - dico = { - k: getattr(self, k) - for k in set(self.__table__.columns.keys()) - .union(self.fields_api_read()) - .union(self.fields_base_read()) - } - elif role == "api": - dico = {k: getattr(self, k) for k in self.fields_api_read()} - else: - dico = {k: getattr(self, k) for k in self.fields_base_read()} - if hasattr(self, "__dump__"): - dico.update(self.__dump__) - for key, value in dico.items(): # preventing association proxy to die - if isinstance(value, _AssociationList): - dico[key] = list(value) - return dico diff --git a/newspipe/web/models/role.py b/newspipe/web/models/role.py deleted file mode 100644 index 4a0bd42d..00000000 --- a/newspipe/web/models/role.py +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -from bootstrap import db - - -class Role(db.Model): - """ - Represent a role. - """ - - id = db.Column(db.Integer, primary_key=True) - name = db.Column(db.String(), unique=True) - - user_id = db.Column(db.Integer, db.ForeignKey("user.id")) diff --git a/newspipe/web/models/tag.py b/newspipe/web/models/tag.py deleted file mode 100644 index 9bd8afc5..00000000 --- a/newspipe/web/models/tag.py +++ /dev/null @@ -1,44 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from bootstrap import db - - -class ArticleTag(db.Model): - text = db.Column(db.String, primary_key=True, unique=False) - - # foreign keys - article_id = db.Column( - db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True - ) - - # relationships - article = db.relationship( - "Article", back_populates="tag_objs", foreign_keys=[article_id] - ) - - def __init__(self, text): - self.text = text - - -class BookmarkTag(db.Model): - id = db.Column(db.Integer, primary_key=True) - text = db.Column(db.String, unique=False) - - # foreign keys - user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE")) - bookmark_id = db.Column( - db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE") - ) - - # relationships - bookmark = db.relationship( - "Bookmark", - back_populates="tags", - cascade="all,delete", - foreign_keys=[bookmark_id], - ) - - # def __init__(self, text, user_id): - # self.text = text - # self.user_id = user_id diff --git a/newspipe/web/models/user.py b/newspipe/web/models/user.py deleted file mode 100644 index f96c8ccb..00000000 --- a/newspipe/web/models/user.py +++ /dev/null @@ -1,114 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# newspipe - A Web based news aggregator. -# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information: https://git.sr.ht/~cedric/newspipe -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import re -import random -import hashlib -from datetime import datetime -from werkzeug.security import check_password_hash -from flask_login import UserMixin -from sqlalchemy.orm import validates - -from bootstrap import db -from web.models.right_mixin import RightMixin -from web.models.category import Category -from web.models.feed import Feed - - -class User(db.Model, UserMixin, RightMixin): - """ - Represent a user. - """ - - id = db.Column(db.Integer, primary_key=True) - nickname = db.Column(db.String(), unique=True) - pwdhash = db.Column(db.String()) - - automatic_crawling = db.Column(db.Boolean(), default=True) - - is_public_profile = db.Column(db.Boolean(), default=False) - bio = db.Column(db.String(5000), default="") - webpage = db.Column(db.String(), default="") - twitter = db.Column(db.String(), default="") - - date_created = db.Column(db.DateTime(), default=datetime.utcnow) - last_seen = db.Column(db.DateTime(), default=datetime.utcnow) - - # user rights - is_active = db.Column(db.Boolean(), default=False) - is_admin = db.Column(db.Boolean(), default=False) - is_api = db.Column(db.Boolean(), default=False) - - # relationships - categories = db.relationship( - "Category", - backref="user", - cascade="all, delete-orphan", - foreign_keys=[Category.user_id], - ) - feeds = db.relationship( - "Feed", - backref="user", - cascade="all, delete-orphan", - foreign_keys=[Feed.user_id], - ) - - @staticmethod - def _fields_base_write(): - return {"login", "password"} - - @staticmethod - def _fields_base_read(): - return {"date_created", "last_connection"} - - @staticmethod - def make_valid_nickname(nickname): - return re.sub("[^a-zA-Z0-9_\.]", "", nickname) - - @validates("bio") - def validates_bio(self, key, value): - assert len(value) <= 5000, AssertionError("maximum length for bio: 5000") - return value.strip() - - def get_id(self): - """ - Return the id of the user. - """ - return self.id - - def check_password(self, password): - """ - Check the password of the user. - """ - return check_password_hash(self.pwdhash, password) - - def __eq__(self, other): - return self.id == other.id - - def __repr__(self): - return "" % (self.nickname) diff --git a/newspipe/web/static/css/custom.css b/newspipe/web/static/css/custom.css deleted file mode 100644 index c85cc8a1..00000000 --- a/newspipe/web/static/css/custom.css +++ /dev/null @@ -1,80 +0,0 @@ -html { - position: relative; - min-height: 100%; - font-size: 16px; -} -body { - /* Margin bottom by footer height */ - margin-bottom: 60px; - background-color: rgb(246, 248, 250); -} -img { - padding: 2px; -} -a { - color: #3572B0; -} - -#sidebar { - overflow: auto; - max-height: 700px; -} - -#sidebar .nav li { - list-style-type: none; - padding: 0; - line-height: 1.0; - flex-direction: row; - flex-wrap: nowrap; /* assumes you only want one row */ -} - -.navbar-dark .navbar-nav .nav-link { - color: - rgb(255, 255, 255, 1); -} - -.bg-newspipe-blue { - /* background-color: #0082c9; */ - background-image: linear-gradient(40deg, #0082c9 0%, #30b6ff 100%); -} -.btn-primary { - background-color: #0082c9; - opacity: 1; - color: white; -} - -.btn-outline-primary { - border-color: #0082c9; - color: #0082c9; -} - -/* Sticky footer */ -.footer { - position: absolute; - bottom: 0; - width: 100%; - /* Set the fixed height of the footer here */ - height: 60px; - line-height: 60px; /* Vertically center the text there */ - background-color: #006FBA; -} -.footer a{ - color: #FFFFFF; -} - -.input-group-inline { - min-width: 0; - width: 200px; - display: inline; -} - -.alert-message { - position: relative; - display: block; - z-index: 1001; -} - -/*customization for Flask-Paginate*/ -.pagination-page-info { - display: inline; -} diff --git a/newspipe/web/static/img/favicon.ico b/newspipe/web/static/img/favicon.ico deleted file mode 100644 index 5b056c1e..00000000 Binary files a/newspipe/web/static/img/favicon.ico and /dev/null differ diff --git a/newspipe/web/static/img/newspipe.png b/newspipe/web/static/img/newspipe.png deleted file mode 100644 index c3ba5029..00000000 Binary files a/newspipe/web/static/img/newspipe.png and /dev/null differ diff --git a/newspipe/web/static/img/newspipe.svg b/newspipe/web/static/img/newspipe.svg deleted file mode 100644 index be22ae42..00000000 --- a/newspipe/web/static/img/newspipe.svg +++ /dev/null @@ -1,84 +0,0 @@ - - - - -Created by potrace 1.13, written by Peter Selinger 2001-2015 - - - - - - - - - - - - - - - - - - - diff --git a/newspipe/web/static/img/pinboard.png b/newspipe/web/static/img/pinboard.png deleted file mode 100644 index 6dddc10b..00000000 Binary files a/newspipe/web/static/img/pinboard.png and /dev/null differ diff --git a/newspipe/web/static/img/reddit.png b/newspipe/web/static/img/reddit.png deleted file mode 100755 index 2d615f2a..00000000 Binary files a/newspipe/web/static/img/reddit.png and /dev/null differ diff --git a/newspipe/web/static/img/twitter.png b/newspipe/web/static/img/twitter.png deleted file mode 100644 index fc11c4ce..00000000 Binary files a/newspipe/web/static/img/twitter.png and /dev/null differ diff --git a/newspipe/web/static/js/articles.js b/newspipe/web/static/js/articles.js deleted file mode 100644 index de579532..00000000 --- a/newspipe/web/static/js/articles.js +++ /dev/null @@ -1,191 +0,0 @@ -/*! -* Newspipe - A Web based news aggregator. -* Copyright (C) 2010-2020 Cédric Bonhomme - https://cedricbonhomme.org -* -* For more information: https://git.sr.ht/~cedric/newspipe -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License as -* published by the Free Software Foundation, either version 3 of the -* License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . - */ - -API_ROOT = '/api/v2.0/' - -if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } - -function change_unread_counter(feed_id, increment) { - var new_value = parseInt($("#unread-"+feed_id).text()) + increment; - $("#unread-"+feed_id).text(new_value); - $("#total-unread").text(parseInt($("#total-unread").text()) + increment); - if (new_value == 0) { - $("#unread-"+feed_id).hide(); - } else { - $("#unread-"+feed_id).show(); - } -} - -+function ($) { - - // Mark an article as read when it is opened in a new table - $('.open-article').on('click', function(e) { - var feed_id = $(this).parent().parent().attr("data-feed"); - var filter = $('#filters').attr("data-filter"); - if (filter == "unread") { - $(this).parent().parent().remove(); - change_unread_counter(feed_id, -1); - } - }); - - - - // Mark an article as read or unread. - $('.readed').on('click', function() { - var article_id = $(this).parent().parent().parent().attr("data-article"); - var feed_id = $(this).parent().parent().parent().attr("data-feed"); - var filter = $('#filters').attr("data-filter"); - - var data; - if ($(this).hasClass('fa-square-o')) { - data = JSON.stringify({ - readed: false - }) - if (filter == "read") { - $(this).parent().parent().parent().remove(); - } - else { - // here, filter == "all" - $(this).parent().parent().parent().children("td:nth-child(2)").css( "font-weight", "bold" ); - $(this).removeClass('fa-square-o').addClass('fa-check-square-o'); - } - change_unread_counter(feed_id, 1); - } - else { - data = JSON.stringify({readed: true}) - if (filter == "unread") { - $(this).parent().parent().parent().remove(); - } - else { - // here, filter == "all" - $(this).parent().parent().parent().children("td:nth-child(2)").css( "font-weight", "normal" ); - $(this).removeClass('fa-check-square-o').addClass('fa-square-o'); - } - change_unread_counter(feed_id, -1); - } - - // sends the updates to the server - $.ajax({ - type: 'PUT', - // Provide correct Content-Type, so that Flask will know how to process it. - contentType: 'application/json', - // Encode your data as JSON. - data: data, - // This is the type of data you're expecting back from the server. - url: API_ROOT + "article/" + article_id, - success: function (result) { - //console.log(result); - }, - error: function(XMLHttpRequest, textStatus, errorThrown){ - console.log(XMLHttpRequest.responseText); - } - }); - }); - - - - // Like or unlike an article - $('.like').on('click', function() { - var article_id = $(this).parent().parent().parent().attr("data-article"); - var data; - if ($(this).hasClass("glyphicon-star")) { - data = JSON.stringify({like: false}) - $(this).removeClass('glyphicon-star').addClass('glyphicon-star-empty'); - if(window.location.pathname.indexOf('/favorites') != -1) { - $(this).parent().parent().parent().remove(); - } - } - else { - data = JSON.stringify({like: true}) - $(this).removeClass('glyphicon-star-empty').addClass('glyphicon-star'); - } - - // sends the updates to the server - $.ajax({ - type: 'PUT', - // Provide correct Content-Type, so that Flask will know how to process it. - contentType: 'application/json', - // Encode your data as JSON. - data: data, - // This is the type of data you're expecting back from the server. - url: API_ROOT + "article/" + article_id, - success: function (result) { - //console.log(result); - }, - error: function(XMLHttpRequest, textStatus, errorThrown){ - console.log(XMLHttpRequest.responseText); - } - }); - }); - - - - // Delete an article - $('.delete').on('click', function() { - var feed_id = $(this).parent().parent().parent().attr("data-feed"); - var article_id = $(this).parent().parent().parent().attr("data-article"); - $(this).parent().parent().parent().remove(); - - // sends the updates to the server - $.ajax({ - type: 'DELETE', - url: API_ROOT + "article/" + article_id, - success: function (result) { - change_unread_counter(feed_id, -1); - }, - error: function(XMLHttpRequest, textStatus, errorThrown){ - console.log(XMLHttpRequest.responseText); - } - }); - }); - - - - // Delete all duplicate articles (used in the page /duplicates) - $('.delete-all').click(function(){ - var data = []; - - var columnNo = $(this).parent().index(); - $(this).closest("table") - .find("tr td:nth-child(" + (columnNo+1) + ")") - .each(function(line, column) { - data.push(parseInt(column.id)); - }).remove(); - - data = JSON.stringify(data); - - // sends the updates to the server - $.ajax({ - type: 'DELETE', - // Provide correct Content-Type, so that Flask will know how to process it. - contentType: 'application/json', - data: data, - url: API_ROOT + "articles", - success: function (result) { - //console.log(result); - }, - error: function(XMLHttpRequest, textStatus, errorThrown){ - console.log(XMLHttpRequest.responseText); - } - }); - - }); - -}(jQuery); diff --git a/newspipe/web/static/js/feed.js b/newspipe/web/static/js/feed.js deleted file mode 100644 index ceef58fc..00000000 --- a/newspipe/web/static/js/feed.js +++ /dev/null @@ -1,22 +0,0 @@ -$('.container').on('click', '#add-feed-filter-row', function() { - $('#filters-container').append( - '
' - + ' ' - + ' ' - + ' ' - + ' ' - + ' ' - + '
'); -}); -$('.container').on('click', '.del-feed-filter-row', function() { - $(this).parent().remove(); -}); diff --git a/newspipe/web/templates/about.html b/newspipe/web/templates/about.html deleted file mode 100644 index 43d8c73d..00000000 --- a/newspipe/web/templates/about.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-

{{ _('About') }}

-

- {{ _('Newspipe is a news aggregator platform.') }} -

{{ _('This software is under AGPLv3 license. You are welcome to copy, modify or - redistribute the source code - according to the Affero GPL license.') }}

-

{{ _('Found a bug? Report it here.') }}

-

{{ _('More information') }} {{ _('about this instance.') }}

-
-
-
-
-
-

{{ _('Help') }}

-

{{ _('Contact')}}: {{ contact }}

-

{{ _('You can subscribe to new feeds with a bookmarklet. Drag the following button to your browser bookmarks.') }}

- {{ _('Subscribe to this feed using Newspipe', bookmarklet='javascript:window.location="%s?url="+encodeURIComponent(document.location)' % url_for('feed.bookmarklet', _external=True)) }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/about_more.html b/newspipe/web/templates/about_more.html deleted file mode 100644 index 94a01c07..00000000 --- a/newspipe/web/templates/about_more.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
    -
  • {{ _('Newspipe version') }}: {{newspipe_version}}
    • -
  • {{ _('Registration') }}: {{registration}}
    • -
  • {{ _('Python version') }}: {{python_version}}
    • -
  • {{ _('Number of users') }}: {{nb_users}}
    • -
-
-{% endblock %} diff --git a/newspipe/web/templates/admin/create_user.html b/newspipe/web/templates/admin/create_user.html deleted file mode 100644 index 8844f987..00000000 --- a/newspipe/web/templates/admin/create_user.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "layout.html" %} -{% block head%} -{{super()}} -{% endblock %} -{% block content %} -
-
-

{{ message | safe }}

-
- {{ form.hidden_tag() }} - - {{ form.nickname.label }} - {{ form.nickname(class_="form-control") }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} - - {{ form.password.label }} - {{ form.password(class_="form-control") }} {% for error in form.password.errors %} {{ error }}
{% endfor %} - - {{ form.automatic_crawling.label }} - {{ form.automatic_crawling(class_="form-control") }} {% for error in form.automatic_crawling.errors %} {{ error }}
{% endfor %} - -
- {{ form.submit(class_="btn btn-primary") }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/admin/dashboard.html b/newspipe/web/templates/admin/dashboard.html deleted file mode 100644 index 8bcc2db0..00000000 --- a/newspipe/web/templates/admin/dashboard.html +++ /dev/null @@ -1,68 +0,0 @@ -{% extends "layout.html" %} -{% block head%} -{{super()}} -{% endblock %} -{% block content %} -
-

{{ _('Registered users') }}

- - - - - - - - - - - - {% for user in users %} - - - - - - - - {% endfor %} - -
#{{ _('Nickname') }}{{ _('Member since') }}{{ _('Last seen') }}{{ _('Actions') }}
{{ loop.index }} - {% if user.is_public_profile %} - {{ user.nickname }} - {% else %} - {{ user.nickname }} - {% endif %} - {% if user.id == current_user.id %} (It's you!){% endif %} - {{ user.date_created | datetime }}{{ user.last_seen | datetime }} - - {% if user.id != current_user.id %} - - {% if user.is_active %} - - {% else %} - - {% endif %} - - - {% endif %} -
-{{ _('Add a new user') }} -
- -{% endblock %} diff --git a/newspipe/web/templates/article.html b/newspipe/web/templates/article.html deleted file mode 100644 index 884bf677..00000000 --- a/newspipe/web/templates/article.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ article.title|safe }}

-

{{ _('from') }} {{ article.source.title }}

- - {% if article.like %} - - {% else %} - - {% endif %} - {% if article.readed %} - - {% else %} - - {% endif %} -
{{ article.date | datetime }}
-
-
- {{ article.content | safe }} -
-
- - - - - - - - - -
-
-{% endblock %} diff --git a/newspipe/web/templates/article_pub.html b/newspipe/web/templates/article_pub.html deleted file mode 100644 index e810d18f..00000000 --- a/newspipe/web/templates/article_pub.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ article.title|safe }}

-

{{ _('from') }} {{ article.source.title }}

-
{{ article.date | datetime }}
-
-
- {{ article.content | safe }} -
-
- - - - - - - - - -
-
-{% endblock %} diff --git a/newspipe/web/templates/bookmarks.html b/newspipe/web/templates/bookmarks.html deleted file mode 100644 index 0f6e02c4..00000000 --- a/newspipe/web/templates/bookmarks.html +++ /dev/null @@ -1,81 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
- {{ pagination.info }} -
-
- {% if current_user.is_authenticated %} - all ⸱  - private ⸱  - public ⸱  - unread - {% endif %} -
-
-
-
-
- {% if tag %} -   {{ tag }} - {% endif %} - {% if query %} - {% if tag %}
{% endif %} -   {{ query }} - {% endif %} -
-
-
-
-
-
- -
- -
-
-
-
-
-
-
-
-
-
- {{ pagination.links }} -
-
- -
-
-
    - {% for bookmark in bookmarks %} -
  • - -

    - {{ bookmark.title }} -

    -

    -

    {{ bookmark.description }}
    -
    {% for tag in bookmark.tags %}{{ tag.text }} {% endfor %}
    - {{ bookmark.time | datetime }} - {% if current_user.is_authenticated %} - edit - delete - {% endif %} -

    - -
    • - {% endfor %} -
-
-
-
-
-
- {{ pagination.links }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/categories.html b/newspipe/web/templates/categories.html deleted file mode 100644 index ea5388a3..00000000 --- a/newspipe/web/templates/categories.html +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ _("You have %(categories)d categories · Add a %(start_link)scategory%(end_link)s", categories=categories|count, start_link=("" % url_for("category.form"))|safe, end_link=""|safe) }}

- {% if categories|count == 0 %} -

{{_("No category")}}

- {% else %} -
- - - - - - - - - - - - {% for category in categories %} - - - - - - - - {% endfor %} - -
#{{ _('Name') }}{{ _('Feeds') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }}{{ category.name }}{{ feeds_count.get(category.id, 0) }}( {{ unread_article_count.get(category.id, 0) }} ) {{ article_count.get(category.id, 0) }} - - -
-
- {% endif %} -{% endblock %} diff --git a/newspipe/web/templates/duplicates.html b/newspipe/web/templates/duplicates.html deleted file mode 100644 index 38dc52b1..00000000 --- a/newspipe/web/templates/duplicates.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ _('Duplicates in the feed') }} {{ feed.title }}.

-

- - - - - - - - - - {% for pair in duplicates %} - - - - - - {% endfor %} - -
# - {{ _('Delete all in this column') }} - - {{ _('Delete all in this column') }} -
{{ loop.index }} {{ pair[0].title }} ({{ pair[0].retrieved_date }}) {{ pair[1].title }} ({{ pair[1].retrieved_date }})
-
-
-{% endblock %} diff --git a/newspipe/web/templates/edit_bookmark.html b/newspipe/web/templates/edit_bookmark.html deleted file mode 100644 index ee0e0243..00000000 --- a/newspipe/web/templates/edit_bookmark.html +++ /dev/null @@ -1,84 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ action }}

-
- {{ form.hidden_tag() }} -
- -
- {{ form.href(class_="form-control", size="100%") }} -
- {% for error in form.href.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.title(class_="form-control", size="100%") }} -
- {% for error in form.title.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.description(class_="form-control", size="100%") }} -
- {% for error in form.description.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.tags(class_="form-control", size="100%") }} -
- {% for error in form.tags.errors %} {{ error }}
{% endfor %} -
- -
- -
-
- {{ form.shared(class_="checkbox", style="margin-left: 0px;") }} -
-
-
- -
- -
-
- {{ form.to_read(class_="checkbox", style="margin-left: 0px;") }} -
-
-
- -
-
- {{ form.submit(class_="btn btn-primary") }} -
-
-
-
- {% if action == _('Add a new bookmark') %} -
-
-

{{ _('You can add a bookmark with a bookmarklet. Drag the following button to your browser bookmarks.') }}

- {{ _('Bookmark this page using Newspipe', bookmarklet='javascript:window.location="%s?href="+encodeURIComponent(document.location)+"&title="+document.title' % url_for('bookmark.bookmarklet', _external=True)) }} -
-
-
-

{{ _('Import bookmarks from Pinboard') }} (*.json)

- - -
- - -
-
-
- {% endif %} -
-{% endblock %} diff --git a/newspipe/web/templates/edit_category.html b/newspipe/web/templates/edit_category.html deleted file mode 100644 index 955d17b9..00000000 --- a/newspipe/web/templates/edit_category.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ action }}

-
- {{ form.hidden_tag() }} -
- -
- {{ form.name(class_="form-control", size="100%") }} -
- {% for error in form.name.errors %} {{ error }}
{% endfor %} -
-
-
- {{ form.submit(class_="btn btn-primary") }} -
-
-
-
-
-{% endblock %} diff --git a/newspipe/web/templates/edit_feed.html b/newspipe/web/templates/edit_feed.html deleted file mode 100644 index a439c78d..00000000 --- a/newspipe/web/templates/edit_feed.html +++ /dev/null @@ -1,98 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ action }}

-
- {{ form.hidden_tag() }} -
- -
- {{ form.link(class_="form-control", size="100%") }} -
- {% for error in form.link.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.title(class_="form-control", size="100%", placeholder=_('Optional')) }} -
- {% for error in form.title.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.site_link(class_="form-control", size="100%", placeholder=_('Optional')) }} -
- {% for error in form.site_link.errors %} {{ error }}
{% endfor %} -
- -
- -
- {{ form.category_id(class_="form-control", placeholder=_('Optional')) }} -
- {% for error in form.category_id.errors %} {{ error }}
{% endfor %} -
- -
- -
-
- {{ form.enabled(class_="checkbox", style="margin-left: 0px;") }} -
-
-
- -
- -
-
- {{ form.private(class_="checkbox", style="margin-left: 0px;") }} -
- {{ _("If checked, articles of this feed won't be available to others and the feed won't be listed on your profile page.", url=url_for('user.profile_public', nickname=current_user.nickname) ) }} - {{ _("Check this box if there is a private token in the link of the feed.") }} -
- -
- -
- -
- -
-
-
- {% if feed %} - {% for filter_ in feed.filters or [] %} -
- - - - - -
- {% endfor %} - {% endif %} -
- -
-
- {{ form.submit(class_="btn btn-primary") }} -
-
-
-
-
-{% endblock %} diff --git a/newspipe/web/templates/emails/account_activation.txt b/newspipe/web/templates/emails/account_activation.txt deleted file mode 100644 index c7d9c52e..00000000 --- a/newspipe/web/templates/emails/account_activation.txt +++ /dev/null @@ -1,9 +0,0 @@ -Hello {{ user.nickname }}, - -Your account has been created. -Click on the following link in order to confirm it: -{{ platform_url }}user/confirm_account/{{ token }} - -The link expires at {{ expire_time.strftime('%Y-%m-%d %H:%M') }}. - -See you, diff --git a/newspipe/web/templates/emails/new_password.txt b/newspipe/web/templates/emails/new_password.txt deleted file mode 100644 index 1a04a36d..00000000 --- a/newspipe/web/templates/emails/new_password.txt +++ /dev/null @@ -1,8 +0,0 @@ -Hello {{ user.nickname }}, - -A new password has been generated at your request: -{{ password }} - -It is advised to replace it as soon as connected to Newspipe. - -See you, diff --git a/newspipe/web/templates/errors/404.html b/newspipe/web/templates/errors/404.html deleted file mode 100644 index c64a2be8..00000000 --- a/newspipe/web/templates/errors/404.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "layout.html" %} -{% block head %} -{{ super() }} -{% endblock %} -{% block content %} -
-
-

Page Not Found

-

What you were looking for is just not there, go to the home page.

-
-
-{% endblock %} diff --git a/newspipe/web/templates/errors/500.html b/newspipe/web/templates/errors/500.html deleted file mode 100644 index 417fc0c7..00000000 --- a/newspipe/web/templates/errors/500.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "layout.html" %} -{% block head %} -{{ super() }} -{% endblock %} -{% block content %} -
-
-

Internal Server Error

-

Something bad just happened! Go to the home page.

-
-
-{% endblock %} diff --git a/newspipe/web/templates/feed.html b/newspipe/web/templates/feed.html deleted file mode 100644 index 4246669b..00000000 --- a/newspipe/web/templates/feed.html +++ /dev/null @@ -1,76 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-

{{ feed.title }}

- {% if feed.description %}

{{ feed.description }}

{% endif %} - {% if current_user.is_authenticated %} - - - {% endif %} -
-
-

- {{ _('This feed contains') }} {{ feed.articles.all()|count }} {{ _('articles') }}.
- {% if category %} - {{ _('This feed is part of category %(category_name)s', category_name=category.name) }}
- {% endif %} - {{ _('Address of the feed') }}: {{ feed.link }}
- {% if feed.site_link != "" %} - {{ _('Address of the site') }}: {{ feed.site_link }}
- {% endif %} - -
- - {% if feed.last_retrieved %} - {{ _("Last download:") }} {{ feed.last_retrieved | datetime }}
- {% endif %} - - {% if feed.error_count >= conf.DEFAULT_MAX_ERROR %} - {{ _("That feed has encountered too much consecutive errors and won't be retrieved anymore.") }}
- {{ _("You can click here to reset the error count and reactivate the feed.", reset_error_url=url_for("feed.reset_errors", feed_id=feed.id)) }} - {% elif feed.error_count > 0 %} - {{ _("The download of this feed has encountered some problems. However its error counter will be reinitialized at the next successful retrieving.") }}
- {% endif %} - - {% if feed.last_error %} - {{ _("Here's the last error encountered while retrieving this feed:") }} 

{{ feed.last_error }}

- {% endif %} - - {% if feed.articles.all()|count != 0 %} - {{ _('The last article was posted') }} {{ elapsed.days }} {{ _('day(s) ago.') }}
- {{ _('Daily average') }}: {{ average }}, {{ _('between the') }} {{ first_post_date | datetime }} {{ _('and the') }} {{ end_post_date | datetime }}. - {% endif %} -

-
- -
-
-
- - - - - - - - - {% for article in articles %} - - - - - {% endfor %} - -
{{ _('Article') }}{{ _('Date') }}
{{ article.title }}{{ article.date | datetime }}
-
-
-
- -
-
- {{ pagination.links }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/feed_list.html b/newspipe/web/templates/feed_list.html deleted file mode 100644 index 11bd65a6..00000000 --- a/newspipe/web/templates/feed_list.html +++ /dev/null @@ -1,55 +0,0 @@ -{% if feeds.count() != 0 %} -
- - - - - - - - - - - - - {% for feed in feeds %} - - - - - - - - - {% endfor %} - -
#{{ _('Status') }}{{ _('Title') }}{{ _('Site') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }} - {% if feed.enabled %} - - {% else %} - - {% endif %} - {% if feed.error_count >= conf.DEFAULT_MAX_ERROR %} - - {% endif %} - {% if feed.icon_url %} {% endif %}{{ feed.title }}{{ feed.site_link }}( {{ unread_article_count.get(feed.id, 0) }} ) {{ article_count.get(feed.id, 0) }} - - - - -
-
- -{% endif %} diff --git a/newspipe/web/templates/feed_list_per_categories.html b/newspipe/web/templates/feed_list_per_categories.html deleted file mode 100644 index 34d10ddd..00000000 --- a/newspipe/web/templates/feed_list_per_categories.html +++ /dev/null @@ -1,52 +0,0 @@ -
-
-
-
- - - -
-
-
-
- -
- -
- - - - - - - - - - {% for feed in feeds %} - - - - - - {% endfor %} - -
#{{ _('Title') }}{{ _('Site') }}
{{ loop.index }}{% if feed.icon_url %} {% endif %} {{ feed.title }}{{ feed.site_link }}
-
- diff --git a/newspipe/web/templates/feed_list_simple.html b/newspipe/web/templates/feed_list_simple.html deleted file mode 100644 index 5f692a53..00000000 --- a/newspipe/web/templates/feed_list_simple.html +++ /dev/null @@ -1,35 +0,0 @@ -{% if feeds | length != 0 %} -
- - - - - - - - - - {% for feed in feeds %} - - - - - - {% endfor %} - -
#{{ _('Title') }}{{ _('Site') }}
{{ loop.index }}{% if feed.icon_url %} {% endif %} {{ feed.title }}{{ feed.site_link }}
-
- -{% endif %} diff --git a/newspipe/web/templates/feeds.html b/newspipe/web/templates/feeds.html deleted file mode 100644 index 805e1b74..00000000 --- a/newspipe/web/templates/feeds.html +++ /dev/null @@ -1,7 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ _('You are subscribed to %(feed_count)d feeds.', feed_count=feeds.count()) }} {{ _('Add') }} {{ _('a feed') }}.

- {% include "feed_list.html" %} -
-{% endblock %} diff --git a/newspipe/web/templates/history.html b/newspipe/web/templates/history.html deleted file mode 100644 index ba567106..00000000 --- a/newspipe/web/templates/history.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ _('History') }}

- {% if month != None %} -

{{ year }}

-

{{ month | month_name }}

- {% elif year != None %} -

 {{ _('all years') }}

-

{{ year }}

- {% endif %} -
    - {% for article in articles_counter | sort(reverse = True) %} - {% if year == None %} -
  • {{ article }} : {{ articles_counter[article] }} articles
    • - {% elif month == None %} -
  • {{ article | month_name }} : {{ articles_counter[article] }} articles
    • - {% else %} - {% for article in articles %} -
  • {{ article.date | datetime }} - {{ article.title | safe }}
    • - {% endfor %} - {% endif %} - {% endfor %} -
-
-{% endblock %} diff --git a/newspipe/web/templates/home.html b/newspipe/web/templates/home.html deleted file mode 100644 index 8da8dbe1..00000000 --- a/newspipe/web/templates/home.html +++ /dev/null @@ -1,135 +0,0 @@ -{% extends "layout.html" %} -{% block content %} - -
-{% if feeds|count == 0 %} -
-

{{ _("You don't have any feeds.") }}

-

{{ _('Add some') }}, {{ _('or') }} {{ _('upload an OPML file.') }}

-
-{% else %} -
- -
- - {% if articles | count != 0%} -
- - - - - - - - - - - {% for article in articles %} - - - - - - - {% endfor %} - -
{{ _('Feed') }}{{ _('Article') }}{{ _('Date') }}
- - {% if article.like %} - - {% else %} - - {% endif %} - {% if article.readed %} - - {% else %} - - {% if filter_ == 'all' %}{% endif %} - {% endif %} - {{ article.source.title | safe }} - {{ article.title | safe }} - {{ article.date | datetime }}
-
- {% endif %} -
-
-{% endif %} -
-{% endblock %} diff --git a/newspipe/web/templates/inactives.html b/newspipe/web/templates/inactives.html deleted file mode 100644 index e89a5fe1..00000000 --- a/newspipe/web/templates/inactives.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-

{{ _('Days of inactivity') }}:

- -
-
- {% if inactives != [] %} - - {% else %} -

{{ _('No inactive feeds.') }}

- {% endif %} -

-
-{% endblock %} diff --git a/newspipe/web/templates/layout.html b/newspipe/web/templates/layout.html deleted file mode 100644 index b40b540c..00000000 --- a/newspipe/web/templates/layout.html +++ /dev/null @@ -1,140 +0,0 @@ - - - - {% block head %} - - - - - Newspipe{% if head_titles %} - {{ ' - '.join(head_titles) }}{% endif %} - - - - - - - - - - - - {% endblock %} - - - {% block menu %} - - {% endblock %} -
- -
- {% block messages %} - {% with messages = get_flashed_messages(with_categories=true) %} - {% if messages %} - {% for category, message in messages %} -
- - {{ message }} -
- {% endfor %} - {% endif %} - {% endwith %} - {% endblock %} -
- - {% block content %}{% endblock %} - - - - - - diff --git a/newspipe/web/templates/login.html b/newspipe/web/templates/login.html deleted file mode 100644 index 1253e2d3..00000000 --- a/newspipe/web/templates/login.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-
-
-

{{ _('Log In') }}

-
- {{ form.hidden_tag() }} -
- {{ form.nickmane(class_="form-control", placeholder=_('Your nickname')) }} -
- {% for message in form.nickmane.errors %} - - {% endfor %} -
- {{ form.password(class_="form-control", placeholder=_('Your Password')) }} -
- {% for message in form.password.errors %} - - {% endfor %} - {{ form.submit(class_="btn btn-primary") }} -
-
-
-
- {{ _('Sign up') }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/management.html b/newspipe/web/templates/management.html deleted file mode 100644 index 22ebde8a..00000000 --- a/newspipe/web/templates/management.html +++ /dev/null @@ -1,99 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-
-
-

{{ _('Your subscriptions') }}

-

{{ _('You are subscribed to') }} {{ nb_feeds }} {{ _('feeds') }}. {{ _('Add') }} {{ _('a feed') }}.

-

{{ nb_articles }} {{ _('articles are stored in the database with') }} {{ nb_unread_articles }} {{ _('unread articles') }}.

-

{{ _('You have') }} {{ nb_categories }} {{ _('categories') }}.

- {{ _('Delete articles older than 10 weeks') }} -
-
-

{{ _('Your bookmarks') }}

-

{{ _('You have') }} {{ nb_bookmarks }} {{ _('bookmarks') }}.

- {{ _('Delete all bookmarks') }} -
-
-
-
-
-
-
-
-
-

{{ _('Your data') }}

-
-
-
-
-
-

{{ _('Articles') }}

-
-
-
-
-
{{ _('Import') }}
-

{{ _('Import a Newspipe account (*.json).') }}

-
- - -
-
-
-
{{ _('Export') }}
-

{{ _('Export your newspipe account to JSON.') }}

- {{ _('Export') }} -
-
-
-
-
-

{{ _('Feeds') }}

-
-
-
-
-
{{ _('Import') }}
-

{{ _('Import feeds from OPML (*.xml or *.opml).') }}

-
- - -
-
-
-
{{ _('Export') }}
-

{{ _('Export feeds to OPML.') }}

-
-
-
- - -
-
- - -
-
- - -
-
- -
-
-
- -
-
-
-

{{ _('Bookmarks') }}

- {{ _('Export your bookmarks to JSON') }} -
-
-
-
-
-{% endblock %} diff --git a/newspipe/web/templates/opml.xml b/newspipe/web/templates/opml.xml deleted file mode 100644 index 7159e279..00000000 --- a/newspipe/web/templates/opml.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - Feeds of {{ user.nickname }} - {{ now | datetime }} - {{ now | datetime }} - {{ user.nickname }} - - - {% for feed in feeds %} - {% endfor %} - diff --git a/newspipe/web/templates/popular.html b/newspipe/web/templates/popular.html deleted file mode 100644 index a80cee57..00000000 --- a/newspipe/web/templates/popular.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-

{{ _('Popular feeds') }}

- all ‧ last year ‧ last month -
-
-
-
- -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/profile.html b/newspipe/web/templates/profile.html deleted file mode 100644 index edbae368..00000000 --- a/newspipe/web/templates/profile.html +++ /dev/null @@ -1,68 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ _('Your Profile') }}

-
-
-

{{ _('Member since') }}: {{ user.date_created | datetime }}.

-

{{ _('Last seen') }}: {{ user.last_seen | datetime }}.

-
-
-
-
-
-
-

Edit your profile

-
-
-
-
- {{ form.hidden_tag() }} - -
- {{ form.nickname.label }} - {{ form.nickname(class_="form-control") }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} - - {{ form.password.label }} - {{ form.password(class_="form-control") }} {% for error in form.password.errors %} {{ error }}
{% endfor %} - - {{ form.password_conf.label }} - {{ form.password_conf(class_="form-control") }} {% for error in form.password_conf.errors %} {{ error }}
{% endfor %} -
- -
- {{ form.bio.label }} - {{ form.bio(class_="form-control") }} {% for error in form.bio.errors %} {{ error }}
{% endfor %} - - {{ form.webpage.label }} - {{ form.webpage(class_="form-control") }} {% for error in form.webpage.errors %} {{ error }}
{% endfor %} - - {{ form.twitter.label }} - {{ form.twitter(class_="form-control") }} {% for error in form.twitter.errors %} {{ error }}
{% endfor %} - - {{ form.is_public_profile.label }} - {{ form.is_public_profile(class_="form-control") }} {% for error in form.is_public_profile.errors %} {{ error }}
{% endfor %} -

{{ _('Your profile will be available here.', url=url_for('user.profile_public', nickname=user.nickname) ) }}

- - {{ form.automatic_crawling.label }} - {{ form.automatic_crawling(class_="form-control") }} {% for error in form.automatic_crawling.errors %} {{ error }}
{% endfor %} -

{{ _('Uncheck if you are using your own crawler.') }}

-
-
-
-
-
- {{ form.submit(class_="btn btn-primary") }} -
-
-
-
-
-
-
-
- {{ _('Delete your account') }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/profile_public.html b/newspipe/web/templates/profile_public.html deleted file mode 100644 index 9ba3578c..00000000 --- a/newspipe/web/templates/profile_public.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-

{{ user.nickname }} / stream

-
-
-

-   - {{ _('Member since') }}: {{ user.date_created | datetime }} -

-

-   - {{ _('Last seen') }}: {{ user.last_seen | datetime }} -

- {% if user.webpage %} -

-   - {{ _('Webpage') }}: {{ user.webpage | safe }} -

- {% endif %} - {% if user.twitter %} -

-   - {{ _('Twitter') }}: {{ user.twitter | safe }} -

- {% endif %} -
-
-
-
- {% if user.bio %} -

{{ user.bio }}

- {% endif %} -
-
-
- -

{{ _('Feeds') }}

-
-
- {% include "feed_list_per_categories.html" %} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/signup.html b/newspipe/web/templates/signup.html deleted file mode 100644 index 8d34b3bf..00000000 --- a/newspipe/web/templates/signup.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
- {{ form.hidden_tag() }} -
- {{ form.nickname(class_="form-control", placeholder=_('Your nickname')) }} {% for error in form.nickname.errors %} {{ error }}
{% endfor %} -

{{ _('Letters, numbers, dots and underscores only.') }}

-
-
- {{ form.email(class_="form-control", placeholder=_('Your email')) }} {% for error in form.email.errors %} {{ error }}
{% endfor %} -

{{ _("Only for account activation. Your email won't be stored.") }}

-
-
- {{ form.password(class_="form-control", placeholder=_('Your password')) }} {% for error in form.password.errors %} {{ error }}
{% endfor %} -

{{ _('Minimum 6 characters.') }}

-
-
- {{ form.submit(class_="btn btn-default") }} -
-
-
-{% endblock %} diff --git a/newspipe/web/templates/user_stream.html b/newspipe/web/templates/user_stream.html deleted file mode 100644 index b05376a8..00000000 --- a/newspipe/web/templates/user_stream.html +++ /dev/null @@ -1,70 +0,0 @@ -{% extends "layout.html" %} -{% block content %} -
-
-
-
-
- - - -
-
-
-
- -

- - {% if category %} -
-
-

Articles from the category {{ category.name }}

-
-
- {% endif %} - -
-
- {{ pagination.info }} -
-
- -
-
- {{ pagination.links }} -
-
- -
- - - - - - - - - - {% for article in articles %} - - - - - - {% endfor %} - -
#{{ _('Title') }}{{ _('Published at') }}
{{ loop.index }}{{ article.title }}{{ article.date }}
-
- -
-
- {{ pagination.links }} -
-
-
-{% endblock %} diff --git a/newspipe/web/views/__init__.py b/newspipe/web/views/__init__.py index a2f7bd1c..7d73ac8c 100644 --- a/newspipe/web/views/__init__.py +++ b/newspipe/web/views/__init__.py @@ -1,37 +1,9 @@ -from web.views.api import v2 -from web.views import views, home, session_mgmt -from web.views.article import article_bp, articles_bp -from web.views.feed import feed_bp, feeds_bp -from web.views.category import category_bp, categories_bp -from web.views.icon import icon_bp -from web.views.admin import admin_bp -from web.views.user import user_bp, users_bp -from web.views.bookmark import bookmark_bp, bookmarks_bp - -__all__ = [ - "views", - "home", - "session_mgmt", - "v2", - "article_bp", - "articles_bp", - "feed_bp", - "feeds_bp", - "category_bp", - "categories_bp", - "icon_bp", - "admin_bp", - "user_bp", - "users_bp", - "bookmark_bp", - "bookmarks_bp", -] - -import conf -from flask import request -from flask import g - - -@g.babel.localeselector -def get_locale(): - return request.accept_languages.best_match(conf.LANGUAGES.keys()) +from newspipe.web.views.api import v2 +from newspipe.web.views import views, home, session_mgmt +from newspipe.web.views.article import article_bp, articles_bp +from newspipe.web.views.feed import feed_bp, feeds_bp +from newspipe.web.views.category import category_bp, categories_bp +from newspipe.web.views.icon import icon_bp +from newspipe.web.views.admin import admin_bp +from newspipe.web.views.user import user_bp, users_bp +from newspipe.web.views.bookmark import bookmark_bp, bookmarks_bp diff --git a/newspipe/web/views/admin.py b/newspipe/web/views/admin.py index fe1b389b..7df683a7 100644 --- a/newspipe/web/views/admin.py +++ b/newspipe/web/views/admin.py @@ -3,10 +3,10 @@ from flask import Blueprint, render_template, redirect, flash, url_for from flask_babel import gettext, format_timedelta from flask_login import login_required, current_user -from lib.utils import redirect_url -from web.views.common import admin_permission -from web.controllers import UserController -from web.forms import InformationMessageForm, UserForm +from newspipe.lib.utils import redirect_url +from newspipe.controllers import UserController +from newspipe.web.views.common import admin_permission +from newspipe.web.forms import InformationMessageForm, UserForm admin_bp = Blueprint("admin", __name__, url_prefix="/admin") diff --git a/newspipe/web/views/api/v2/__init__.py b/newspipe/web/views/api/v2/__init__.py index ef587e72..d367da20 100644 --- a/newspipe/web/views/api/v2/__init__.py +++ b/newspipe/web/views/api/v2/__init__.py @@ -1,3 +1,3 @@ -from web.views.api.v2 import article, feed, category +from newspipe.web.views.api.v2 import article, feed, category __all__ = ["article", "feed", "category"] diff --git a/newspipe/web/views/api/v2/article.py b/newspipe/web/views/api/v2/article.py index 8da6c6dd..1dbf23b2 100644 --- a/newspipe/web/views/api/v2/article.py +++ b/newspipe/web/views/api/v2/article.py @@ -1,12 +1,12 @@ -from conf import API_ROOT +from newspipe.bootstrap import application import dateutil.parser from datetime import datetime from flask import current_app from flask_restful import Api -from web.views.common import api_permission -from web.controllers import ArticleController -from web.views.api.v2.common import ( +from newspipe.web.views.common import api_permission +from newspipe.controllers import ArticleController +from newspipe.web.views.api.v2.common import ( PyAggAbstractResource, PyAggResourceNew, PyAggResourceExisting, @@ -49,7 +49,7 @@ class ArticlesChallenge(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json") api.add_resource(ArticleAPI, "/article/", endpoint="article.json") diff --git a/newspipe/web/views/api/v2/category.py b/newspipe/web/views/api/v2/category.py index a830624d..5f7ef354 100644 --- a/newspipe/web/views/api/v2/category.py +++ b/newspipe/web/views/api/v2/category.py @@ -1,9 +1,9 @@ -from conf import API_ROOT from flask import current_app from flask_restful import Api -from web.controllers.category import CategoryController -from web.views.api.v2.common import ( +from newspipe.bootstrap import application +from newspipe.controllers.category import CategoryController +from newspipe.web.views.api.v2.common import ( PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti, @@ -22,7 +22,7 @@ class CategoriesAPI(PyAggResourceMulti): controller_cls = CategoryController -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json") api.add_resource(CategoryAPI, "/category/", endpoint="category.json") api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json") diff --git a/newspipe/web/views/api/v2/common.py b/newspipe/web/views/api/v2/common.py index 81248422..3d90bf91 100644 --- a/newspipe/web/views/api/v2/common.py +++ b/newspipe/web/views/api/v2/common.py @@ -26,13 +26,13 @@ from flask import request from flask_restful import Resource, reqparse from flask_login import current_user -from web.views.common import ( +from newspipe.web.views.common import ( admin_permission, api_permission, login_user_bundle, jsonify, ) -from web.controllers import UserController +from newspipe.controllers import UserController logger = logging.getLogger(__name__) diff --git a/newspipe/web/views/api/v2/feed.py b/newspipe/web/views/api/v2/feed.py index 1e4fabf2..1081ed8c 100644 --- a/newspipe/web/views/api/v2/feed.py +++ b/newspipe/web/views/api/v2/feed.py @@ -1,11 +1,11 @@ -from conf import API_ROOT from flask import current_app from flask_restful import Api -from web.views.common import api_permission -from web.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT +from newspipe.bootstrap import application +from newspipe.web.views.common import api_permission +from newspipe.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from web.views.api.v2.common import ( +from newspipe.web.views.api.v2.common import ( PyAggAbstractResource, PyAggResourceNew, PyAggResourceExisting, @@ -39,7 +39,7 @@ class FetchableFeedAPI(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json") api.add_resource(FeedAPI, "/feed/", endpoint="feed.json") diff --git a/newspipe/web/views/article.py b/newspipe/web/views/article.py index c0c6f346..a49859ca 100644 --- a/newspipe/web/views/article.py +++ b/newspipe/web/views/article.py @@ -14,11 +14,11 @@ from flask_babel import gettext from flask_login import login_required, current_user -from bootstrap import db -from lib.utils import clear_string, redirect_url -from lib.data import export_json -from web.controllers import ArticleController, UserController, CategoryController -from web.lib.view_utils import etag_match +from newspipe.bootstrap import db +from newspipe.lib.utils import clear_string, redirect_url +from newspipe.lib.data import export_json +from newspipe.controllers import ArticleController, UserController, CategoryController +from newspipe.web.lib.view_utils import etag_match articles_bp = Blueprint("articles", __name__, url_prefix="/articles") article_bp = Blueprint("article", __name__, url_prefix="/article") diff --git a/newspipe/web/views/bookmark.py b/newspipe/web/views/bookmark.py index 2577f747..ea49b3c8 100644 --- a/newspipe/web/views/bookmark.py +++ b/newspipe/web/views/bookmark.py @@ -44,13 +44,12 @@ from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args from sqlalchemy import desc -import conf -from lib.utils import redirect_url -from lib.data import import_pinboard_json, export_bookmarks -from bootstrap import db -from web.forms import BookmarkForm -from web.controllers import BookmarkController, BookmarkTagController -from web.models import BookmarkTag +from newspipe.lib.utils import redirect_url +from newspipe.lib.data import import_pinboard_json, export_bookmarks +from newspipe.bootstrap import db +from newspipe.web.forms import BookmarkForm +from newspipe.controllers import BookmarkController, BookmarkTagController +from newspipe.models import BookmarkTag logger = logging.getLogger(__name__) bookmarks_bp = Blueprint("bookmarks", __name__, url_prefix="/bookmarks") diff --git a/newspipe/web/views/category.py b/newspipe/web/views/category.py index 1c897058..9f1a7e83 100644 --- a/newspipe/web/views/category.py +++ b/newspipe/web/views/category.py @@ -2,10 +2,10 @@ from flask import Blueprint, render_template, flash, redirect, url_for from flask_babel import gettext from flask_login import login_required, current_user -from web.forms import CategoryForm -from lib.utils import redirect_url -from web.lib.view_utils import etag_match -from web.controllers import ArticleController, FeedController, CategoryController +from newspipe.web.forms import CategoryForm +from newspipe.lib.utils import redirect_url +from newspipe.web.lib.view_utils import etag_match +from newspipe.controllers import ArticleController, FeedController, CategoryController categories_bp = Blueprint("categories", __name__, url_prefix="/categories") category_bp = Blueprint("category", __name__, url_prefix="/category") diff --git a/newspipe/web/views/common.py b/newspipe/web/views/common.py index c2d8e2df..8d9ecfa9 100644 --- a/newspipe/web/views/common.py +++ b/newspipe/web/views/common.py @@ -10,8 +10,8 @@ from flask_principal import ( session_identity_loader, identity_changed, ) -from web.controllers import UserController -from lib.utils import default_handler +from newspipe.controllers import UserController +from newspipe.lib.utils import default_handler admin_role = RoleNeed("admin") api_role = RoleNeed("api") diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py index 592e3cbf..0be30668 100644 --- a/newspipe/web/views/feed.py +++ b/newspipe/web/views/feed.py @@ -17,12 +17,12 @@ from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args -import conf -from lib import misc_utils, utils -from lib.feed_utils import construct_feed_from -from web.lib.view_utils import etag_match -from web.forms import AddFeedForm -from web.controllers import ( +from newspipe.bootstrap import application +from newspipe.lib import misc_utils, utils +from newspipe.lib.feed_utils import construct_feed_from +from newspipe.web.lib.view_utils import etag_match +from newspipe.web.forms import AddFeedForm +from newspipe.controllers import ( UserController, CategoryController, FeedController, @@ -179,7 +179,7 @@ def bookmarklet(): ) feed = feed_contr.create(**feed) flash(gettext("Feed was successfully created."), "success") - if feed.enabled and conf.CRAWLING_METHOD == "default": + if feed.enabled and application.confg['CRAWLING_METHOD'] == "default": misc_utils.fetch(current_user.id, feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=feed.id)) @@ -286,7 +286,7 @@ def process_form(feed_id=None): "success", ) - if conf.CRAWLING_METHOD == "default": + if application.confg['CRAWLING_METHOD'] == "default": misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), "info") @@ -335,7 +335,7 @@ def export(): if not include_private: filter["private"] = False if not include_exceeded_error_count: - filter["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filter["error_count__lt"] = application.confg['DEFAULT_MAX_ERROR'] user = UserController(current_user.id).get(id=current_user.id) feeds = FeedController(current_user.id).read(**filter) diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py index 1cfa3601..66ce87ea 100644 --- a/newspipe/web/views/home.py +++ b/newspipe/web/views/home.py @@ -7,13 +7,13 @@ from flask_login import login_required, current_user from flask_babel import gettext, get_locale from babel.dates import format_datetime, format_timedelta -import conf -from lib.utils import redirect_url -from lib import misc_utils -from web.lib.view_utils import etag_match -from web.views.common import jsonify +from newspipe.bootstrap import application +from newspipe.lib.utils import redirect_url +from newspipe.lib import misc_utils +from newspipe.web.lib.view_utils import etag_match +from newspipe.web.views.common import jsonify -from web.controllers import FeedController, ArticleController, CategoryController +from newspipe.controllers import FeedController, ArticleController, CategoryController localize = pytz.utc.localize logger = logging.getLogger(__name__) @@ -181,7 +181,7 @@ def fetch(feed_id=None): Triggers the download of news. News are downloaded in a separated process. """ - if conf.CRAWLING_METHOD == "default" and current_user.is_admin: + if application.config['CRAWLING_METHOD'] == "default" and current_user.is_admin: misc_utils.fetch(current_user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: diff --git a/newspipe/web/views/icon.py b/newspipe/web/views/icon.py index e1de6402..4cdcd4b0 100644 --- a/newspipe/web/views/icon.py +++ b/newspipe/web/views/icon.py @@ -1,7 +1,8 @@ import base64 from flask import Blueprint, Response, request -from web.controllers import IconController -from web.lib.view_utils import etag_match + +from newspipe.controllers import IconController +from newspipe.web.lib.view_utils import etag_match icon_bp = Blueprint("icon", __name__, url_prefix="/icon") diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py index 809825d3..9bdb89cb 100644 --- a/newspipe/web/views/session_mgmt.py +++ b/newspipe/web/views/session_mgmt.py @@ -24,11 +24,11 @@ from flask_principal import ( session_identity_loader, ) -import conf -from web.views.common import admin_role, api_role, login_user_bundle -from web.controllers import UserController -from web.forms import SignupForm, SigninForm -from notifications import notifications +from newspipe.bootstrap import application +from newspipe.web.views.common import admin_role, api_role, login_user_bundle +from newspipe.controllers import UserController +from newspipe.web.forms import SignupForm, SigninForm +from newspipe.notifications import notifications Principal(current_app) # Create a permission with a single Need, in this case a RoleNeed. @@ -99,7 +99,7 @@ def logout(): @current_app.route("/signup", methods=["GET", "POST"]) def signup(): - if not conf.SELF_REGISTRATION: + if not application.config['SELF_REGISTRATION']: flash(gettext("Self-registration is disabled."), "warning") return redirect(url_for("home")) if current_user.is_authenticated: diff --git a/newspipe/web/views/user.py b/newspipe/web/views/user.py index 10974947..39eefb4c 100644 --- a/newspipe/web/views/user.py +++ b/newspipe/web/views/user.py @@ -6,12 +6,12 @@ from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args -import conf -from notifications import notifications -from lib import misc_utils -from lib.data import import_opml, import_json -from web.lib.user_utils import confirm_token -from web.controllers import ( +from newspipe.bootstrap import application +from newspipe.notifications import notifications +from newspipe.lib import misc_utils +from newspipe.lib.data import import_opml, import_json +from newspipe.web.lib.user_utils import confirm_token +from newspipe.controllers import ( UserController, FeedController, ArticleController, @@ -19,7 +19,7 @@ from web.controllers import ( BookmarkController, ) -from web.forms import ProfileForm +from newspipe.web.forms import ProfileForm users_bp = Blueprint("users", __name__, url_prefix="/users") user_bp = Blueprint("user", __name__, url_prefix="/user") @@ -115,7 +115,7 @@ def management(): else: try: nb = import_opml(current_user.nickname, data.read()) - if conf.CRAWLING_METHOD == "classic": + if application.config['CRAWLING_METHOD'] == "classic": misc_utils.fetch(current_user.id, None) flash(str(nb) + " " + gettext("feeds imported."), "success") flash(gettext("Downloading articles..."), "info") diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py index e08270cd..6aa6ce54 100644 --- a/newspipe/web/views/views.py +++ b/newspipe/web/views/views.py @@ -6,11 +6,10 @@ from flask import request, render_template, flash, url_for, redirect, current_ap from flask_babel import gettext from sqlalchemy import desc -import conf -from web import __version__ -from conf import API_ROOT, ADMIN_EMAIL -from web.controllers import FeedController, UserController -from web.lib.view_utils import etag_match +from newspipe.bootstrap import application +from newspipe.web import __version__ +from newspipe.controllers import FeedController, UserController +from newspipe.web.lib.view_utils import etag_match logger = logging.getLogger(__name__) @@ -25,7 +24,7 @@ def authentication_required(error): @current_app.errorhandler(403) def authentication_failed(error): - if API_ROOT in request.url: + if application.conf['API_ROOT'] in request.url: return error flash(gettext("Forbidden."), "danger") return redirect(url_for("login")) @@ -71,7 +70,7 @@ def popular(): filters = {} filters["created_date__gt"] = not_added_before filters["private"] = False - filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] feeds = FeedController().count_by_link(**filters) sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True) return render_template("popular.html", popular=sorted_feeds) @@ -80,7 +79,7 @@ def popular(): @current_app.route("/about", methods=["GET"]) @etag_match def about(): - return render_template("about.html", contact=ADMIN_EMAIL) + return render_template("about.html", contact=application.config['ADMIN_EMAIL']) @current_app.route("/about/more", methods=["GET"]) @@ -102,7 +101,7 @@ def about_more(): "about_more.html", newspipe_version=newspipe_version, version_url=version_url, - registration=[conf.SELF_REGISTRATION and "Open" or "Closed"][0], + registration=[application.config['SELF_REGISTRATION'] and "Open" or "Closed"][0], python_version="{}.{}.{}".format(*sys.version_info[:3]), nb_users=UserController().read().count(), ) diff --git a/package-lock.json b/package-lock.json index f17279a2..106e200a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "resolved": "https://registry.npmjs.org/datatables/-/datatables-1.10.18.tgz", "integrity": "sha512-ntatMgS9NN6UMpwbmO+QkYJuKlVeMA2Mi0Gu/QxyIh+dW7ZjLSDhPT2tWlzjpIWEkDYgieDzS9Nu7bdQCW0sbQ==", "requires": { - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "datatables.net": { @@ -22,7 +22,7 @@ "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-1.10.20.tgz", "integrity": "sha512-4E4S7tTU607N3h0fZPkGmAtr9mwy462u+VJ6gxYZ8MxcRIjZqHy3Dv1GNry7i3zQCktTdWbULVKBbkAJkuHEnQ==", "requires": { - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "datatables.net-bs4": { @@ -31,7 +31,7 @@ "integrity": "sha512-kQmMUMsHMOlAW96ztdoFqjSbLnlGZQ63iIM82kHbmldsfYdzuyhbb4hTx6YNBi481WCO3iPSvI6YodNec46ZAw==", "requires": { "datatables.net": "1.10.20", - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "fork-awesome": { diff --git a/package.json b/package.json index 099771b0..e89900cd 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,6 @@ "yarn": ">= 1.0.0" }, "scripts": { - "postinstall": "cd newspipe/web/static/ ; ln -sf ../../../node_modules npm_components" + "postinstall": "cd newspipe/static/ ; ln -sf ../../node_modules npm_components" } } diff --git a/runserver.py b/runserver.py new file mode 100755 index 00000000..20c712a4 --- /dev/null +++ b/runserver.py @@ -0,0 +1,68 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information: https://git.sr.ht/~cedric/newspipe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import calendar +from newspipe.bootstrap import application +from flask_babel import Babel, format_datetime + + +babel = Babel(application) + + +# Jinja filters +def month_name(month_number): + return calendar.month_name[month_number] + + +application.jinja_env.filters["month_name"] = month_name +application.jinja_env.filters["datetime"] = format_datetime +# inject application in Jinja env +application.jinja_env.globals["application"] = application + +# Views +from flask_restful import Api +from flask import g + +with application.app_context(): + g.api = Api(application, prefix="/api/v2.0") + g.babel = babel + + from newspipe.web import views + + application.register_blueprint(views.articles_bp) + application.register_blueprint(views.article_bp) + application.register_blueprint(views.feeds_bp) + application.register_blueprint(views.feed_bp) + application.register_blueprint(views.categories_bp) + application.register_blueprint(views.category_bp) + application.register_blueprint(views.icon_bp) + application.register_blueprint(views.admin_bp) + application.register_blueprint(views.users_bp) + application.register_blueprint(views.user_bp) + application.register_blueprint(views.bookmarks_bp) + application.register_blueprint(views.bookmark_bp) + + +if __name__ == "__main__": + application.run( + host=application.config['HOST'], + port=application.config['PORT'], + debug=application.config['DEBUG'] + ) -- cgit