diff options
author | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2020-03-09 23:16:05 +0100 |
---|---|---|
committer | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2020-03-09 23:16:05 +0100 |
commit | 3ab6290d4994b33cdbf831523938cdb18a13bf49 (patch) | |
tree | 685980f53aaa3eda4e27ddfc7032554f55528e57 | |
parent | Improved method to detect current version of the Newspipe instance. (diff) | |
download | newspipe-3ab6290d4994b33cdbf831523938cdb18a13bf49.tar.gz newspipe-3ab6290d4994b33cdbf831523938cdb18a13bf49.tar.bz2 newspipe-3ab6290d4994b33cdbf831523938cdb18a13bf49.zip |
Refactoring the backend.
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | instance/production.py | 56 | ||||
-rwxr-xr-x | manager.py (renamed from newspipe/manager.py) | 8 | ||||
-rw-r--r-- | newspipe/bootstrap.py | 49 | ||||
-rw-r--r-- | newspipe/conf.py | 2 | ||||
-rw-r--r-- | newspipe/conf/conf.cfg-sample | 31 | ||||
-rw-r--r-- | newspipe/controllers/__init__.py (renamed from newspipe/web/controllers/__init__.py) | 0 | ||||
-rw-r--r-- | newspipe/controllers/abstract.py (renamed from newspipe/web/controllers/abstract.py) | 2 | ||||
-rw-r--r-- | newspipe/controllers/article.py (renamed from newspipe/web/controllers/article.py) | 8 | ||||
-rw-r--r-- | newspipe/controllers/bookmark.py (renamed from newspipe/web/controllers/bookmark.py) | 4 | ||||
-rw-r--r-- | newspipe/controllers/category.py (renamed from newspipe/web/controllers/category.py) | 2 | ||||
-rw-r--r-- | newspipe/controllers/feed.py (renamed from newspipe/web/controllers/feed.py) | 8 | ||||
-rw-r--r-- | newspipe/controllers/icon.py (renamed from newspipe/web/controllers/icon.py) | 2 | ||||
-rw-r--r-- | newspipe/controllers/tag.py (renamed from newspipe/web/controllers/tag.py) | 4 | ||||
-rw-r--r-- | newspipe/controllers/user.py (renamed from newspipe/web/controllers/user.py) | 2 | ||||
-rw-r--r-- | newspipe/crawler/default_crawler.py | 24 | ||||
-rw-r--r-- | newspipe/lib/article_utils.py | 10 | ||||
-rw-r--r-- | newspipe/lib/data.py | 8 | ||||
-rw-r--r-- | newspipe/lib/feed_utils.py | 6 | ||||
-rwxr-xr-x | newspipe/lib/misc_utils.py | 10 | ||||
-rw-r--r-- | newspipe/lib/utils.py | 10 | ||||
-rw-r--r-- | newspipe/models/__init__.py (renamed from newspipe/web/models/__init__.py) | 0 | ||||
-rw-r--r-- | newspipe/models/article.py (renamed from newspipe/web/models/article.py) | 4 | ||||
-rw-r--r-- | newspipe/models/bookmark.py (renamed from newspipe/web/models/bookmark.py) | 6 | ||||
-rw-r--r-- | newspipe/models/category.py (renamed from newspipe/web/models/category.py) | 4 | ||||
-rw-r--r-- | newspipe/models/feed.py (renamed from newspipe/web/models/feed.py) | 6 | ||||
-rw-r--r-- | newspipe/models/icon.py (renamed from newspipe/web/models/icon.py) | 2 | ||||
-rw-r--r-- | newspipe/models/right_mixin.py (renamed from newspipe/web/models/right_mixin.py) | 0 | ||||
-rw-r--r-- | newspipe/models/role.py (renamed from newspipe/web/models/role.py) | 2 | ||||
-rw-r--r-- | newspipe/models/tag.py (renamed from newspipe/web/models/tag.py) | 2 | ||||
-rw-r--r-- | newspipe/models/user.py (renamed from newspipe/web/models/user.py) | 8 | ||||
-rw-r--r-- | newspipe/notifications/emails.py | 16 | ||||
-rw-r--r-- | newspipe/notifications/notifications.py | 15 | ||||
-rw-r--r-- | newspipe/static/css/custom.css (renamed from newspipe/web/static/css/custom.css) | 0 | ||||
-rw-r--r-- | newspipe/static/img/favicon.ico (renamed from newspipe/web/static/img/favicon.ico) | bin | 1150 -> 1150 bytes | |||
-rw-r--r-- | newspipe/static/img/newspipe.png (renamed from newspipe/web/static/img/newspipe.png) | bin | 1547 -> 1547 bytes | |||
-rw-r--r-- | newspipe/static/img/newspipe.svg (renamed from newspipe/web/static/img/newspipe.svg) | 0 | ||||
-rw-r--r-- | newspipe/static/img/pinboard.png (renamed from newspipe/web/static/img/pinboard.png) | bin | 597 -> 597 bytes | |||
-rwxr-xr-x | newspipe/static/img/reddit.png (renamed from newspipe/web/static/img/reddit.png) | bin | 525 -> 525 bytes | |||
-rw-r--r-- | newspipe/static/img/twitter.png (renamed from newspipe/web/static/img/twitter.png) | bin | 640 -> 640 bytes | |||
-rw-r--r-- | newspipe/static/js/articles.js (renamed from newspipe/web/static/js/articles.js) | 0 | ||||
-rw-r--r-- | newspipe/static/js/feed.js (renamed from newspipe/web/static/js/feed.js) | 0 | ||||
-rw-r--r-- | newspipe/templates/about.html (renamed from newspipe/web/templates/about.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/about_more.html (renamed from newspipe/web/templates/about_more.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/admin/create_user.html (renamed from newspipe/web/templates/admin/create_user.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/admin/dashboard.html (renamed from newspipe/web/templates/admin/dashboard.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/article.html (renamed from newspipe/web/templates/article.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/article_pub.html (renamed from newspipe/web/templates/article_pub.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/bookmarks.html (renamed from newspipe/web/templates/bookmarks.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/categories.html (renamed from newspipe/web/templates/categories.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/duplicates.html (renamed from newspipe/web/templates/duplicates.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/edit_bookmark.html (renamed from newspipe/web/templates/edit_bookmark.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/edit_category.html (renamed from newspipe/web/templates/edit_category.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/edit_feed.html (renamed from newspipe/web/templates/edit_feed.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/emails/account_activation.txt (renamed from newspipe/web/templates/emails/account_activation.txt) | 0 | ||||
-rw-r--r-- | newspipe/templates/emails/new_password.txt (renamed from newspipe/web/templates/emails/new_password.txt) | 0 | ||||
-rw-r--r-- | newspipe/templates/errors/404.html (renamed from newspipe/web/templates/errors/404.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/errors/500.html (renamed from newspipe/web/templates/errors/500.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/feed.html (renamed from newspipe/web/templates/feed.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/feed_list.html (renamed from newspipe/web/templates/feed_list.html) | 2 | ||||
-rw-r--r-- | newspipe/templates/feed_list_per_categories.html (renamed from newspipe/web/templates/feed_list_per_categories.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/feed_list_simple.html (renamed from newspipe/web/templates/feed_list_simple.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/feeds.html (renamed from newspipe/web/templates/feeds.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/history.html (renamed from newspipe/web/templates/history.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/home.html (renamed from newspipe/web/templates/home.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/inactives.html (renamed from newspipe/web/templates/inactives.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/layout.html (renamed from newspipe/web/templates/layout.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/login.html (renamed from newspipe/web/templates/login.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/management.html (renamed from newspipe/web/templates/management.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/opml.xml (renamed from newspipe/web/templates/opml.xml) | 0 | ||||
-rw-r--r-- | newspipe/templates/popular.html (renamed from newspipe/web/templates/popular.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/profile.html (renamed from newspipe/web/templates/profile.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/profile_public.html (renamed from newspipe/web/templates/profile_public.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/signup.html (renamed from newspipe/web/templates/signup.html) | 0 | ||||
-rw-r--r-- | newspipe/templates/user_stream.html (renamed from newspipe/web/templates/user_stream.html) | 0 | ||||
-rw-r--r-- | newspipe/web/forms.py | 6 | ||||
-rw-r--r-- | newspipe/web/lib/user_utils.py | 6 | ||||
-rw-r--r-- | newspipe/web/lib/view_utils.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/__init__.py | 46 | ||||
-rw-r--r-- | newspipe/web/views/admin.py | 8 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/__init__.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/article.py | 10 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/category.py | 8 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/common.py | 4 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/feed.py | 10 | ||||
-rw-r--r-- | newspipe/web/views/article.py | 10 | ||||
-rw-r--r-- | newspipe/web/views/bookmark.py | 13 | ||||
-rw-r--r-- | newspipe/web/views/category.py | 8 | ||||
-rw-r--r-- | newspipe/web/views/common.py | 4 | ||||
-rw-r--r-- | newspipe/web/views/feed.py | 18 | ||||
-rw-r--r-- | newspipe/web/views/home.py | 14 | ||||
-rw-r--r-- | newspipe/web/views/icon.py | 5 | ||||
-rw-r--r-- | newspipe/web/views/session_mgmt.py | 12 | ||||
-rw-r--r-- | newspipe/web/views/user.py | 16 | ||||
-rw-r--r-- | newspipe/web/views/views.py | 17 | ||||
-rw-r--r-- | package-lock.json | 6 | ||||
-rw-r--r-- | package.json | 2 | ||||
-rwxr-xr-x | runserver.py (renamed from newspipe/runserver.py) | 12 |
98 files changed, 276 insertions, 282 deletions
@@ -24,10 +24,10 @@ build newspipe.egg-info/ -newspipe/conf/conf.cfg +instance/development.py .coverage # js and node files node_modules -newspipe/web/static/npm_components -newspipe/web/static/js/bundle.min.js +newspipe/static/npm_components +newspipe/static/js/bundle.min.js diff --git a/instance/production.py b/instance/production.py new file mode 100644 index 00000000..af2f1c1e --- /dev/null +++ b/instance/production.py @@ -0,0 +1,56 @@ +# [webserver] +HOST = '127.0.0.1' +PORT = 5000 +SECRET_KEY = 'a secret only you know' +DEBUG = False +TESTING = False +API_ROOT = "/api/v2.0" + +SECRET_KEY = 'LCx3BchmHRxFzkEv4BqQJyeXRLXenf' +SECURITY_PASSWORD_SALT = 'L8gTsyrpRQEF8jNWQPyvRfv7U5kJkD' + + +# [misc] +ADMIN_EMAIL = 'admin@admin.localhost' +SECURITY_PASSWORD_SALT = 'a secret to confirm user account' +TOKEN_VALIDITY_PERIOD = 3600 +LOG_PATH = './var/newspipe.log' +NB_WORKER = 5 +DEBUG = False +TESTING = False +LOG_LEVEL = 'info' +SELF_REGISTRATION = True + + +# [database] +DB_CONFIG_DICT = { + 'user': 'user', + 'password': 'password', + 'host': 'localhost', + 'port': 5432 +} +DATABASE_NAME = 'newspipe' +SQLALCHEMY_DATABASE_URI = 'postgres://{user}:{password}@{host}:{port}/{name}'.format(name=DATABASE_NAME, **DB_CONFIG_DICT) +SQLALCHEMY_TRACK_MODIFICATIONS = False + + +# [crawler] +CRAWLING_METHOD = 'default' +DEFAULT_MAX_ERROR = 3 +HTTP_PROXY = '' +USER_AGENT = 'JARR (https://github.com/JARR/JARR)' +RESOLVE_ARTICLE_URL = False +TIMEOUT = 30 +RESOLV = False +FEED_REFRESH_INTERVAL = 0 + + +# [notification] +MAIL_SERVER = 'localhost' +MAIL_PORT = 25 +MAIL_USE_TLS = False +MAIL_USE_SSL = False +MAIL_DEBUG = DEBUG +MAIL_USERNAME = None +MAIL_PASSWORD = None +MAIL_DEFAULT_SENDER = ADMIN_EMAIL diff --git a/newspipe/manager.py b/manager.py index 60f4c729..bf935632 100755 --- a/newspipe/manager.py +++ b/manager.py @@ -5,12 +5,12 @@ import os import logging from datetime import datetime from werkzeug.security import generate_password_hash -from bootstrap import application, db, conf, set_logging from flask_script import Manager from flask_migrate import Migrate, MigrateCommand -import web.models -from web.controllers import UserController +import newspipe.models +from newspipe.controllers import UserController +from newspipe.bootstrap import application, db, set_logging logger = logging.getLogger("manager") @@ -62,7 +62,7 @@ def fetch_asyncio(user_id=None, feed_id=None): import asyncio with application.app_context(): - from crawler import default_crawler + from newspipe.crawler import default_crawler filters = {} filters["is_active"] = True diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py index cbdbc820..1fd1ab04 100644 --- a/newspipe/bootstrap.py +++ b/newspipe/bootstrap.py @@ -4,9 +4,9 @@ # required imports and code execution for basic functionning import os -import conf import logging from urllib.parse import urlsplit +from flask_babel import Babel, format_datetime def set_logging( @@ -47,41 +47,36 @@ from flask import Flask from flask_sqlalchemy import SQLAlchemy # Create Flask application -application = Flask("web") +application = Flask(__name__, instance_relative_config=True) if os.environ.get("Newspipe_TESTING", False) == "true": application.debug = logging.DEBUG application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" application.config["TESTING"] = True else: - application.debug = conf.LOG_LEVEL <= logging.DEBUG - application.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False - application.config["SQLALCHEMY_DATABASE_URI"] = conf.SQLALCHEMY_DATABASE_URI - if "postgres" in conf.SQLALCHEMY_DATABASE_URI: - application.config["SQLALCHEMY_POOL_SIZE"] = 15 - application.config["SQLALCHEMY_MAX_OVERFLOW"] = 0 + try: + application.config.from_pyfile("development.py", silent=False) + except Exception: + application.config.from_pyfile("production.py", silent=False) -scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL) -application.config["SERVER_NAME"] = domain -application.config["PREFERRED_URL_SCHEME"] = scheme +# scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL) +# application.config["SERVER_NAME"] = domain +# application.config["PREFERRED_URL_SCHEME"] = scheme -set_logging(conf.LOG_PATH, log_level=conf.LOG_LEVEL) - -# Create secrey key so we can use sessions -application.config["SECRET_KEY"] = getattr(conf, "WEBSERVER_SECRET", None) -if not application.config["SECRET_KEY"]: - application.config["SECRET_KEY"] = os.urandom(12) - -application.config["SECURITY_PASSWORD_SALT"] = getattr( - conf, "SECURITY_PASSWORD_SALT", None -) -if not application.config["SECURITY_PASSWORD_SALT"]: - application.config["SECURITY_PASSWORD_SALT"] = os.urandom(12) +set_logging(application.config['LOG_PATH']) db = SQLAlchemy(application) -def populate_g(): - from flask import g +babel = Babel(application) + - g.db = db - g.app = application +@babel.localeselector +def get_locale(): + # if a user is logged in, use the locale from the user settings + # user = getattr(g, 'user', None) + # if user is not None: + # return user.locale + # otherwise try to guess the language from the user accept + # header the browser transmits. We support de/fr/en in this + # example. The best match wins. + return request.accept_languages.best_match(["fr", "en"]) diff --git a/newspipe/conf.py b/newspipe/conf.py index bb51e563..09a38be5 100644 --- a/newspipe/conf.py +++ b/newspipe/conf.py @@ -10,7 +10,7 @@ import logging BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PATH = os.path.abspath(".") -API_ROOT = "/api/v2.0" + # available languages LANGUAGES = {"en": "English", "fr": "French"} diff --git a/newspipe/conf/conf.cfg-sample b/newspipe/conf/conf.cfg-sample deleted file mode 100644 index 41035fc7..00000000 --- a/newspipe/conf/conf.cfg-sample +++ /dev/null @@ -1,31 +0,0 @@ -[webserver] -host = 0.0.0.0 -port = 5000 -secret_key = a secret only you know -debug = true -[cdn] -cdn_address = https://cdn.cedricbonhomme.org/ -[misc] -platform_url = http://127.0.0.1:5000/ -admin_email = -security_password_salt = a secret to confirm user account -token_validity_period = 3600 -log_path = ./var/log/newspipe.log -log_level = info -[database] -database_url = sqlite:///newspipe.db -[crawler] -crawling_method = default -default_max_error = 6 -user_agent = Newspipe (https://git.sr.ht/~cedric/newspipe) -timeout = 30 -resolv = false -feed_refresh_interval = 120 -[notification] -notification_email = Newspipe@no-reply.com -host = smtp.googlemail.com -port = 465 -tls = false -ssl = true -username = your-gmail-username -password = your-gmail-password diff --git a/newspipe/web/controllers/__init__.py b/newspipe/controllers/__init__.py index 9b193cc5..9b193cc5 100644 --- a/newspipe/web/controllers/__init__.py +++ b/newspipe/controllers/__init__.py diff --git a/newspipe/web/controllers/abstract.py b/newspipe/controllers/abstract.py index 9d9e84f2..a699afd0 100644 --- a/newspipe/web/controllers/abstract.py +++ b/newspipe/controllers/abstract.py @@ -1,6 +1,6 @@ import logging import dateutil.parser -from bootstrap import db +from newspipe.bootstrap import db from datetime import datetime from collections import defaultdict from sqlalchemy import or_, func diff --git a/newspipe/web/controllers/article.py b/newspipe/controllers/article.py index d5efcb74..d4adf99d 100644 --- a/newspipe/web/controllers/article.py +++ b/newspipe/controllers/article.py @@ -4,11 +4,11 @@ import sqlalchemy from sqlalchemy import func from collections import Counter -from bootstrap import db +from newspipe.bootstrap import db from .abstract import AbstractController -from lib.article_utils import process_filters -from web.controllers import CategoryController, FeedController -from web.models import Article +from newspipe.lib.article_utils import process_filters +from newspipe.controllers import CategoryController, FeedController +from newspipe.models import Article logger = logging.getLogger(__name__) diff --git a/newspipe/web/controllers/bookmark.py b/newspipe/controllers/bookmark.py index d1c1260c..3c9dcce9 100644 --- a/newspipe/web/controllers/bookmark.py +++ b/newspipe/controllers/bookmark.py @@ -2,8 +2,8 @@ import logging import itertools from datetime import datetime, timedelta -from bootstrap import db -from web.models import Bookmark +from newspipe.bootstrap import db +from newspipe.models import Bookmark from .abstract import AbstractController from .tag import BookmarkTagController diff --git a/newspipe/web/controllers/category.py b/newspipe/controllers/category.py index ec54f5a3..36f1dea2 100644 --- a/newspipe/web/controllers/category.py +++ b/newspipe/controllers/category.py @@ -1,5 +1,5 @@ from .abstract import AbstractController -from web.models import Category +from newspipe.models import Category from .feed import FeedController diff --git a/newspipe/web/controllers/feed.py b/newspipe/controllers/feed.py index 19ba463f..798efa6e 100644 --- a/newspipe/web/controllers/feed.py +++ b/newspipe/controllers/feed.py @@ -2,15 +2,15 @@ import logging import itertools from datetime import datetime, timedelta -import conf +from newspipe.bootstrap import application from .abstract import AbstractController from .icon import IconController -from web.models import User, Feed -from lib.utils import clear_string +from newspipe.models import User, Feed +from newspipe.lib.utils import clear_string logger = logging.getLogger(__name__) DEFAULT_LIMIT = 5 -DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR +DEFAULT_MAX_ERROR = application.config['DEFAULT_MAX_ERROR'] class FeedController(AbstractController): diff --git a/newspipe/web/controllers/icon.py b/newspipe/controllers/icon.py index de86b52f..0fa7a39e 100644 --- a/newspipe/web/controllers/icon.py +++ b/newspipe/controllers/icon.py @@ -1,6 +1,6 @@ import base64 import requests -from web.models import Icon +from newspipe.models import Icon from .abstract import AbstractController diff --git a/newspipe/web/controllers/tag.py b/newspipe/controllers/tag.py index 35fd5613..c97ce20e 100644 --- a/newspipe/web/controllers/tag.py +++ b/newspipe/controllers/tag.py @@ -2,9 +2,9 @@ import logging import itertools from datetime import datetime, timedelta -from bootstrap import db +from newspipe.bootstrap import db from .abstract import AbstractController -from web.models.tag import BookmarkTag +from newspipe.models.tag import BookmarkTag logger = logging.getLogger(__name__) diff --git a/newspipe/web/controllers/user.py b/newspipe/controllers/user.py index 71eb7d08..7396e52c 100644 --- a/newspipe/web/controllers/user.py +++ b/newspipe/controllers/user.py @@ -1,7 +1,7 @@ import logging from werkzeug.security import generate_password_hash, check_password_hash from .abstract import AbstractController -from web.models import User +from newspipe.models import User logger = logging.getLogger(__name__) diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py index 828066ff..9296e5e4 100644 --- a/newspipe/crawler/default_crawler.py +++ b/newspipe/crawler/default_crawler.py @@ -1,8 +1,8 @@ #! /usr/bin/env python # -*- coding: utf-8 - -# newspipe - A Web based news aggregator. -# Copyright (C) 2010-2019 Cédric Bonhomme - https://www.cedricbonhomme.org +# Newspipe - A Web based news aggregator. +# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org # # For more information: https://git.sr.ht/~cedric/newspipe # @@ -34,13 +34,13 @@ import dateutil.parser from datetime import datetime, timezone, timedelta from sqlalchemy import or_ -import conf -from bootstrap import db -from web.models import User -from web.controllers import FeedController, ArticleController -from lib.utils import jarr_get -from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.article_utils import construct_article, extract_id, get_article_content +from newspipe.bootstrap import application +from newspipe.bootstrap import db +from newspipe.models import User +from newspipe.controllers import FeedController, ArticleController +from newspipe.lib.utils import newspipe_get +from newspipe.lib.feed_utils import construct_feed_from, is_parsing_ok +from newspipe.lib.article_utils import construct_article, extract_id, get_article_content logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ async def parse_feed(user, feed): # with (await sem): try: logger.info("Retrieving feed {}".format(feed.link)) - resp = await jarr_get(feed.link, timeout=5) + resp = await newspipe_get(feed.link, timeout=5) except Exception as e: logger.info("Problem when reading feed {}".format(feed.link)) return @@ -163,9 +163,9 @@ async def retrieve_feed(queue, users, feed_id=None): if feed_id is not None: filters["id"] = feed_id filters["enabled"] = True - filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] filters["last_retrieved__lt"] = datetime.now() - timedelta( - minutes=conf.FEED_REFRESH_INTERVAL + minutes=application.config['FEED_REFRESH_INTERVAL'] ) feeds = FeedController().read(**filters).all() diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py index c2494c79..77cea397 100644 --- a/newspipe/lib/article_utils.py +++ b/newspipe/lib/article_utils.py @@ -9,8 +9,8 @@ import dateutil.parser from bs4 import BeautifulSoup, SoupStrainer from requests.exceptions import MissingSchema -import conf -from lib.utils import jarr_get +from newspipe.bootstrap import application +from newspipe.lib.utils import newspipe_get logger = logging.getLogger(__name__) PROCESSED_DATE_KEYS = {"published", "created", "updated"} @@ -77,16 +77,16 @@ def get_article_content(entry): async def get_article_details(entry, fetch=True): article_link = entry.get("link") article_title = html.unescape(entry.get("title", "")) - if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: + if fetch and application.config['CRAWLER_RESOLV'] and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) - response = await jarr_get(article_link, timeout=5) + response = await newspipe_get(article_link, timeout=5) except MissingSchema: split, failed = urlsplit(article_link), False for scheme in "https", "http": new_link = urlunsplit(SplitResult(scheme, *split[1:])) try: - response = await jarr_get(new_link, timeout=5) + response = await newspipe_get(new_link, timeout=5) except Exception as error: failed = True continue diff --git a/newspipe/lib/data.py b/newspipe/lib/data.py index b8cc3c07..dd7a0503 100644 --- a/newspipe/lib/data.py +++ b/newspipe/lib/data.py @@ -35,10 +35,10 @@ import opml import datetime from flask import jsonify -from bootstrap import db -from web.models import User, Feed, Article -from web.models.tag import BookmarkTag -from web.controllers import BookmarkController, BookmarkTagController +from newspipe.bootstrap import db +from newspipe.models import User, Feed, Article +from newspipe.models.tag import BookmarkTag +from newspipe.controllers import BookmarkController, BookmarkTagController def import_opml(nickname, opml_content): diff --git a/newspipe/lib/feed_utils.py b/newspipe/lib/feed_utils.py index 9f1e2354..a4c6c275 100644 --- a/newspipe/lib/feed_utils.py +++ b/newspipe/lib/feed_utils.py @@ -3,10 +3,10 @@ import urllib import logging import requests import feedparser -from conf import CRAWLER_USER_AGENT from bs4 import BeautifulSoup, SoupStrainer -from lib.utils import try_keys, try_get_icon_url, rebuild_url +from newspipe.bootstrap import application +from newspipe.lib.utils import try_keys, try_get_icon_url, rebuild_url logger = logging.getLogger(__name__) logging.captureWarnings(True) @@ -39,7 +39,7 @@ def escape_keys(*keys): @escape_keys("title", "description") def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): - requests_kwargs = {"headers": {"User-Agent": CRAWLER_USER_AGENT}, "verify": False} + requests_kwargs = {"headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, "verify": False} if url is None and fp_parsed is not None: url = fp_parsed.get("url") if url is not None and fp_parsed is None: diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py index ab57037f..b1ccd075 100755 --- a/newspipe/lib/misc_utils.py +++ b/newspipe/lib/misc_utils.py @@ -45,9 +45,9 @@ from collections import Counter from contextlib import contextmanager from flask import request -import conf -from web.controllers import ArticleController -from lib.utils import clear_string +from newspipe.bootstrap import application +from newspipe.controllers import ArticleController +from newspipe.lib.utils import clear_string logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def fetch(id, feed_id=None): """ cmd = [ sys.executable, - conf.BASE_DIR + "/manager.py", + application.config['BASE_DIR'] + "/manager.py", "fetch_asyncio", "--user_id=" + str(id), ] @@ -154,7 +154,7 @@ def load_stop_words(): Load the stop words and return them in a list. """ stop_words_lists = glob.glob( - os.path.join(conf.BASE_DIR, "web/var/stop_words/*.txt") + os.path.join(application.config['BASE_DIR'], "web/var/stop_words/*.txt") ) stop_words = [] diff --git a/newspipe/lib/utils.py b/newspipe/lib/utils.py index f7244e17..958c9f23 100644 --- a/newspipe/lib/utils.py +++ b/newspipe/lib/utils.py @@ -6,7 +6,7 @@ import requests from hashlib import md5 from flask import request, url_for -import conf +from newspipe.bootstrap import application logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def try_get_icon_url(url, *splits): response = None # if html in content-type, we assume it's a fancy 404 page try: - response = jarr_get(rb_url) + response = newspipe_get(rb_url) content_type = response.headers.get("content-type", "") except Exception: pass @@ -89,12 +89,12 @@ def redirect_url(default="home"): return request.args.get("next") or request.referrer or url_for(default) -async def jarr_get(url, **kwargs): +async def newspipe_get(url, **kwargs): request_kwargs = { "verify": False, "allow_redirects": True, - "timeout": conf.CRAWLER_TIMEOUT, - "headers": {"User-Agent": conf.CRAWLER_USER_AGENT}, + "timeout": application.config['CRAWLER_TIMEOUT'], + "headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, } request_kwargs.update(kwargs) return requests.get(url, **request_kwargs) diff --git a/newspipe/web/models/__init__.py b/newspipe/models/__init__.py index a58a7ad5..a58a7ad5 100644 --- a/newspipe/web/models/__init__.py +++ b/newspipe/models/__init__.py diff --git a/newspipe/web/models/article.py b/newspipe/models/article.py index 4e623ba2..ecc2352b 100644 --- a/newspipe/web/models/article.py +++ b/newspipe/models/article.py @@ -26,12 +26,12 @@ __revision__ = "$Date: 2016/10/04 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" -from bootstrap import db +from newspipe.bootstrap import db from datetime import datetime from sqlalchemy import Index from sqlalchemy.ext.associationproxy import association_proxy -from web.models.right_mixin import RightMixin +from newspipe.models.right_mixin import RightMixin class Article(db.Model, RightMixin): diff --git a/newspipe/web/models/bookmark.py b/newspipe/models/bookmark.py index 11b2db53..c3bc2dea 100644 --- a/newspipe/web/models/bookmark.py +++ b/newspipe/models/bookmark.py @@ -26,14 +26,14 @@ __revision__ = "$Date: 2016/12/07 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" -from bootstrap import db +from newspipe.bootstrap import db from datetime import datetime from sqlalchemy import desc from sqlalchemy.orm import validates from sqlalchemy.ext.associationproxy import association_proxy -from web.models.tag import BookmarkTag -from web.models.right_mixin import RightMixin +from newspipe.models.tag import BookmarkTag +from newspipe.models.right_mixin import RightMixin class Bookmark(db.Model, RightMixin): diff --git a/newspipe/web/models/category.py b/newspipe/models/category.py index bb47ce45..3abb6717 100644 --- a/newspipe/web/models/category.py +++ b/newspipe/models/category.py @@ -1,9 +1,9 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- -from bootstrap import db +from newspipe.bootstrap import db from sqlalchemy import Index -from web.models.right_mixin import RightMixin +from newspipe.models.right_mixin import RightMixin class Category(db.Model, RightMixin): diff --git a/newspipe/web/models/feed.py b/newspipe/models/feed.py index 70a5d521..fc88c17f 100644 --- a/newspipe/web/models/feed.py +++ b/newspipe/models/feed.py @@ -26,12 +26,12 @@ __revision__ = "$Date: 2014/04/12 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" -from bootstrap import db +from newspipe.bootstrap import db from datetime import datetime from sqlalchemy import desc, Index from sqlalchemy.orm import validates -from web.models.right_mixin import RightMixin -from web.models.article import Article +from newspipe.models.right_mixin import RightMixin +from newspipe.models.article import Article class Feed(db.Model, RightMixin): diff --git a/newspipe/web/models/icon.py b/newspipe/models/icon.py index adc9cf69..99c10224 100644 --- a/newspipe/web/models/icon.py +++ b/newspipe/models/icon.py @@ -1,7 +1,7 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- -from bootstrap import db +from newspipe.bootstrap import db class Icon(db.Model): diff --git a/newspipe/web/models/right_mixin.py b/newspipe/models/right_mixin.py index 670beafa..670beafa 100644 --- a/newspipe/web/models/right_mixin.py +++ b/newspipe/models/right_mixin.py diff --git a/newspipe/web/models/role.py b/newspipe/models/role.py index 4a0bd42d..628467c0 100644 --- a/newspipe/web/models/role.py +++ b/newspipe/models/role.py @@ -26,7 +26,7 @@ __revision__ = "$Date: 2014/04/12 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" -from bootstrap import db +from newspipe.bootstrap import db class Role(db.Model): diff --git a/newspipe/web/models/tag.py b/newspipe/models/tag.py index 9bd8afc5..b853c344 100644 --- a/newspipe/web/models/tag.py +++ b/newspipe/models/tag.py @@ -1,7 +1,7 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- -from bootstrap import db +from newspipe.bootstrap import db class ArticleTag(db.Model): diff --git a/newspipe/web/models/user.py b/newspipe/models/user.py index f96c8ccb..142536fe 100644 --- a/newspipe/web/models/user.py +++ b/newspipe/models/user.py @@ -34,10 +34,10 @@ from werkzeug.security import check_password_hash from flask_login import UserMixin from sqlalchemy.orm import validates -from bootstrap import db -from web.models.right_mixin import RightMixin -from web.models.category import Category -from web.models.feed import Feed +from newspipe.bootstrap import db +from newspipe.models.right_mixin import RightMixin +from newspipe.models.category import Category +from newspipe.models.feed import Feed class User(db.Model, UserMixin, RightMixin): diff --git a/newspipe/notifications/emails.py b/newspipe/notifications/emails.py index dcfcf771..9738ae95 100644 --- a/newspipe/notifications/emails.py +++ b/newspipe/notifications/emails.py @@ -24,8 +24,8 @@ import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -import conf -from web.decorators import async_maker +from newspipe.bootstrap import application +from newspipe.web.decorators import async_maker logger = logging.getLogger(__name__) @@ -33,8 +33,8 @@ logger = logging.getLogger(__name__) @async_maker def send_async_email(mfrom, mto, msg): try: - s = smtplib.SMTP(conf.NOTIFICATION_HOST) - s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) + s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) + s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) except Exception: logger.exception("send_async_email raised:") else: @@ -56,7 +56,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): # Create message container - the correct MIME type is multipart/alternative. msg = MIMEMultipart("alternative") msg["Subject"] = subject - msg["From"] = conf.NOTIFICATION_EMAIL + msg["From"] = application.config['NOTIFICATION_EMAIL'] msg["To"] = to msg["BCC"] = bcc @@ -71,12 +71,12 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): msg.attach(part2) try: - s = smtplib.SMTP(conf.NOTIFICATION_HOST) - s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) + s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) + s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) except Exception: logger.exception("send_smtp raised:") else: s.sendmail( - conf.NOTIFICATION_EMAIL, msg["To"] + ", " + msg["BCC"], msg.as_string() + application.config['NOTIFICATION_EMAIL'], msg["To"] + ", " + msg["BCC"], msg.as_string() ) s.quit() diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py index 664c19a7..71ab0cf8 100644 --- a/newspipe/notifications/notifications.py +++ b/newspipe/notifications/notifications.py @@ -21,9 +21,10 @@ import datetime from flask import render_template -import conf -from notifications import emails -from web.lib.user_utils import generate_confirmation_token + +from newspipe.bootstrap import application +from newspipe.notifications import emails +from newspipe.web.lib.user_utils import generate_confirmation_token def new_account_notification(user, email): @@ -32,20 +33,20 @@ def new_account_notification(user, email): """ token = generate_confirmation_token(user.nickname) expire_time = datetime.datetime.now() + datetime.timedelta( - seconds=conf.TOKEN_VALIDITY_PERIOD + seconds=application.config['TOKEN_VALIDITY_PERIOD'] ) plaintext = render_template( "emails/account_activation.txt", user=user, - platform_url=conf.PLATFORM_URL, + platform_url=application.config['PLATFORM_URL'], token=token, expire_time=expire_time, ) emails.send( to=email, - bcc=conf.NOTIFICATION_EMAIL, + bcc=application.config['NOTIFICATION_EMAIL'], subject="[Newspipe] Account creation", plaintext=plaintext, ) @@ -58,7 +59,7 @@ def new_password_notification(user, password): plaintext = render_template("emails/new_password.txt", user=user, password=password) emails.send( to=user.email, - bcc=conf.NOTIFICATION_EMAIL, + bcc=application.config['NOTIFICATION_EMAIL'], subject="[Newspipe] New password", plaintext=plaintext, ) diff --git a/newspipe/web/static/css/custom.css b/newspipe/static/css/custom.css index c85cc8a1..c85cc8a1 100644 --- a/newspipe/web/static/css/custom.css +++ b/newspipe/static/css/custom.css diff --git a/newspipe/web/static/img/favicon.ico b/newspipe/static/img/favicon.ico Binary files differindex 5b056c1e..5b056c1e 100644 --- a/newspipe/web/static/img/favicon.ico +++ b/newspipe/static/img/favicon.ico diff --git a/newspipe/web/static/img/newspipe.png b/newspipe/static/img/newspipe.png Binary files differindex c3ba5029..c3ba5029 100644 --- a/newspipe/web/static/img/newspipe.png +++ b/newspipe/static/img/newspipe.png diff --git a/newspipe/web/static/img/newspipe.svg b/newspipe/static/img/newspipe.svg index be22ae42..be22ae42 100644 --- a/newspipe/web/static/img/newspipe.svg +++ b/newspipe/static/img/newspipe.svg diff --git a/newspipe/web/static/img/pinboard.png b/newspipe/static/img/pinboard.png Binary files differindex 6dddc10b..6dddc10b 100644 --- a/newspipe/web/static/img/pinboard.png +++ b/newspipe/static/img/pinboard.png diff --git a/newspipe/web/static/img/reddit.png b/newspipe/static/img/reddit.png Binary files differindex 2d615f2a..2d615f2a 100755 --- a/newspipe/web/static/img/reddit.png +++ b/newspipe/static/img/reddit.png diff --git a/newspipe/web/static/img/twitter.png b/newspipe/static/img/twitter.png Binary files differindex fc11c4ce..fc11c4ce 100644 --- a/newspipe/web/static/img/twitter.png +++ b/newspipe/static/img/twitter.png diff --git a/newspipe/web/static/js/articles.js b/newspipe/static/js/articles.js index de579532..de579532 100644 --- a/newspipe/web/static/js/articles.js +++ b/newspipe/static/js/articles.js diff --git a/newspipe/web/static/js/feed.js b/newspipe/static/js/feed.js index ceef58fc..ceef58fc 100644 --- a/newspipe/web/static/js/feed.js +++ b/newspipe/static/js/feed.js diff --git a/newspipe/web/templates/about.html b/newspipe/templates/about.html index 43d8c73d..43d8c73d 100644 --- a/newspipe/web/templates/about.html +++ b/newspipe/templates/about.html diff --git a/newspipe/web/templates/about_more.html b/newspipe/templates/about_more.html index 94a01c07..94a01c07 100644 --- a/newspipe/web/templates/about_more.html +++ b/newspipe/templates/about_more.html diff --git a/newspipe/web/templates/admin/create_user.html b/newspipe/templates/admin/create_user.html index 8844f987..8844f987 100644 --- a/newspipe/web/templates/admin/create_user.html +++ b/newspipe/templates/admin/create_user.html diff --git a/newspipe/web/templates/admin/dashboard.html b/newspipe/templates/admin/dashboard.html index 8bcc2db0..8bcc2db0 100644 --- a/newspipe/web/templates/admin/dashboard.html +++ b/newspipe/templates/admin/dashboard.html diff --git a/newspipe/web/templates/article.html b/newspipe/templates/article.html index 884bf677..884bf677 100644 --- a/newspipe/web/templates/article.html +++ b/newspipe/templates/article.html diff --git a/newspipe/web/templates/article_pub.html b/newspipe/templates/article_pub.html index e810d18f..e810d18f 100644 --- a/newspipe/web/templates/article_pub.html +++ b/newspipe/templates/article_pub.html diff --git a/newspipe/web/templates/bookmarks.html b/newspipe/templates/bookmarks.html index 0f6e02c4..0f6e02c4 100644 --- a/newspipe/web/templates/bookmarks.html +++ b/newspipe/templates/bookmarks.html diff --git a/newspipe/web/templates/categories.html b/newspipe/templates/categories.html index ea5388a3..ea5388a3 100644 --- a/newspipe/web/templates/categories.html +++ b/newspipe/templates/categories.html diff --git a/newspipe/web/templates/duplicates.html b/newspipe/templates/duplicates.html index 38dc52b1..38dc52b1 100644 --- a/newspipe/web/templates/duplicates.html +++ b/newspipe/templates/duplicates.html diff --git a/newspipe/web/templates/edit_bookmark.html b/newspipe/templates/edit_bookmark.html index ee0e0243..ee0e0243 100644 --- a/newspipe/web/templates/edit_bookmark.html +++ b/newspipe/templates/edit_bookmark.html diff --git a/newspipe/web/templates/edit_category.html b/newspipe/templates/edit_category.html index 955d17b9..955d17b9 100644 --- a/newspipe/web/templates/edit_category.html +++ b/newspipe/templates/edit_category.html diff --git a/newspipe/web/templates/edit_feed.html b/newspipe/templates/edit_feed.html index a439c78d..a439c78d 100644 --- a/newspipe/web/templates/edit_feed.html +++ b/newspipe/templates/edit_feed.html diff --git a/newspipe/web/templates/emails/account_activation.txt b/newspipe/templates/emails/account_activation.txt index c7d9c52e..c7d9c52e 100644 --- a/newspipe/web/templates/emails/account_activation.txt +++ b/newspipe/templates/emails/account_activation.txt diff --git a/newspipe/web/templates/emails/new_password.txt b/newspipe/templates/emails/new_password.txt index 1a04a36d..1a04a36d 100644 --- a/newspipe/web/templates/emails/new_password.txt +++ b/newspipe/templates/emails/new_password.txt diff --git a/newspipe/web/templates/errors/404.html b/newspipe/templates/errors/404.html index c64a2be8..c64a2be8 100644 --- a/newspipe/web/templates/errors/404.html +++ b/newspipe/templates/errors/404.html diff --git a/newspipe/web/templates/errors/500.html b/newspipe/templates/errors/500.html index 417fc0c7..417fc0c7 100644 --- a/newspipe/web/templates/errors/500.html +++ b/newspipe/templates/errors/500.html diff --git a/newspipe/web/templates/feed.html b/newspipe/templates/feed.html index 4246669b..4246669b 100644 --- a/newspipe/web/templates/feed.html +++ b/newspipe/templates/feed.html diff --git a/newspipe/web/templates/feed_list.html b/newspipe/templates/feed_list.html index 11bd65a6..2ce7841f 100644 --- a/newspipe/web/templates/feed_list.html +++ b/newspipe/templates/feed_list.html @@ -21,7 +21,7 @@ {% else %} <i class="fa fa-eye-slash" aria-hidden="true" title="{{ _('Feed disabled') }}"></i> {% endif %} - {% if feed.error_count >= conf.DEFAULT_MAX_ERROR %} + {% if feed.error_count >= application.config['DEFAULT_MAX_ERROR'] %} <i class="fa fa-exclamation" aria-hidden="true" title="{{ _('Feed encountered too much errors.') }}"></i> {% endif %} </td> diff --git a/newspipe/web/templates/feed_list_per_categories.html b/newspipe/templates/feed_list_per_categories.html index 34d10ddd..34d10ddd 100644 --- a/newspipe/web/templates/feed_list_per_categories.html +++ b/newspipe/templates/feed_list_per_categories.html diff --git a/newspipe/web/templates/feed_list_simple.html b/newspipe/templates/feed_list_simple.html index 5f692a53..5f692a53 100644 --- a/newspipe/web/templates/feed_list_simple.html +++ b/newspipe/templates/feed_list_simple.html diff --git a/newspipe/web/templates/feeds.html b/newspipe/templates/feeds.html index 805e1b74..805e1b74 100644 --- a/newspipe/web/templates/feeds.html +++ b/newspipe/templates/feeds.html diff --git a/newspipe/web/templates/history.html b/newspipe/templates/history.html index ba567106..ba567106 100644 --- a/newspipe/web/templates/history.html +++ b/newspipe/templates/history.html diff --git a/newspipe/web/templates/home.html b/newspipe/templates/home.html index 8da8dbe1..8da8dbe1 100644 --- a/newspipe/web/templates/home.html +++ b/newspipe/templates/home.html diff --git a/newspipe/web/templates/inactives.html b/newspipe/templates/inactives.html index e89a5fe1..e89a5fe1 100644 --- a/newspipe/web/templates/inactives.html +++ b/newspipe/templates/inactives.html diff --git a/newspipe/web/templates/layout.html b/newspipe/templates/layout.html index b40b540c..b40b540c 100644 --- a/newspipe/web/templates/layout.html +++ b/newspipe/templates/layout.html diff --git a/newspipe/web/templates/login.html b/newspipe/templates/login.html index 1253e2d3..1253e2d3 100644 --- a/newspipe/web/templates/login.html +++ b/newspipe/templates/login.html diff --git a/newspipe/web/templates/management.html b/newspipe/templates/management.html index 22ebde8a..22ebde8a 100644 --- a/newspipe/web/templates/management.html +++ b/newspipe/templates/management.html diff --git a/newspipe/web/templates/opml.xml b/newspipe/templates/opml.xml index 7159e279..7159e279 100644 --- a/newspipe/web/templates/opml.xml +++ b/newspipe/templates/opml.xml diff --git a/newspipe/web/templates/popular.html b/newspipe/templates/popular.html index a80cee57..a80cee57 100644 --- a/newspipe/web/templates/popular.html +++ b/newspipe/templates/popular.html diff --git a/newspipe/web/templates/profile.html b/newspipe/templates/profile.html index edbae368..edbae368 100644 --- a/newspipe/web/templates/profile.html +++ b/newspipe/templates/profile.html diff --git a/newspipe/web/templates/profile_public.html b/newspipe/templates/profile_public.html index 9ba3578c..9ba3578c 100644 --- a/newspipe/web/templates/profile_public.html +++ b/newspipe/templates/profile_public.html diff --git a/newspipe/web/templates/signup.html b/newspipe/templates/signup.html index 8d34b3bf..8d34b3bf 100644 --- a/newspipe/web/templates/signup.html +++ b/newspipe/templates/signup.html diff --git a/newspipe/web/templates/user_stream.html b/newspipe/templates/user_stream.html index b05376a8..b05376a8 100644 --- a/newspipe/web/templates/user_stream.html +++ b/newspipe/templates/user_stream.html diff --git a/newspipe/web/forms.py b/newspipe/web/forms.py index e1921210..540b2723 100644 --- a/newspipe/web/forms.py +++ b/newspipe/web/forms.py @@ -43,9 +43,9 @@ from wtforms import ( ) from wtforms.fields.html5 import EmailField, URLField -from lib import misc_utils -from web.controllers import UserController -from web.models import User +from newspipe.lib import misc_utils +from newspipe.controllers import UserController +from newspipe.models import User class SignupForm(FlaskForm): diff --git a/newspipe/web/lib/user_utils.py b/newspipe/web/lib/user_utils.py index 84b1c75c..95b436a9 100644 --- a/newspipe/web/lib/user_utils.py +++ b/newspipe/web/lib/user_utils.py @@ -1,6 +1,6 @@ from itsdangerous import URLSafeTimedSerializer -import conf -from bootstrap import application + +from newspipe.bootstrap import application def generate_confirmation_token(nickname): @@ -14,7 +14,7 @@ def confirm_token(token): nickname = serializer.loads( token, salt=application.config["SECURITY_PASSWORD_SALT"], - max_age=conf.TOKEN_VALIDITY_PERIOD, + max_age=application.config['TOKEN_VALIDITY_PERIOD'], ) except: return False diff --git a/newspipe/web/lib/view_utils.py b/newspipe/web/lib/view_utils.py index 218ebb4c..c6e722d3 100644 --- a/newspipe/web/lib/view_utils.py +++ b/newspipe/web/lib/view_utils.py @@ -1,6 +1,6 @@ from functools import wraps from flask import request, Response, make_response -from lib.utils import to_hash +from newspipe.lib.utils import to_hash def etag_match(func): diff --git a/newspipe/web/views/__init__.py b/newspipe/web/views/__init__.py index a2f7bd1c..7d73ac8c 100644 --- a/newspipe/web/views/__init__.py +++ b/newspipe/web/views/__init__.py @@ -1,37 +1,9 @@ -from web.views.api import v2 -from web.views import views, home, session_mgmt -from web.views.article import article_bp, articles_bp -from web.views.feed import feed_bp, feeds_bp -from web.views.category import category_bp, categories_bp -from web.views.icon import icon_bp -from web.views.admin import admin_bp -from web.views.user import user_bp, users_bp -from web.views.bookmark import bookmark_bp, bookmarks_bp - -__all__ = [ - "views", - "home", - "session_mgmt", - "v2", - "article_bp", - "articles_bp", - "feed_bp", - "feeds_bp", - "category_bp", - "categories_bp", - "icon_bp", - "admin_bp", - "user_bp", - "users_bp", - "bookmark_bp", - "bookmarks_bp", -] - -import conf -from flask import request -from flask import g - - -@g.babel.localeselector -def get_locale(): - return request.accept_languages.best_match(conf.LANGUAGES.keys()) +from newspipe.web.views.api import v2 +from newspipe.web.views import views, home, session_mgmt +from newspipe.web.views.article import article_bp, articles_bp +from newspipe.web.views.feed import feed_bp, feeds_bp +from newspipe.web.views.category import category_bp, categories_bp +from newspipe.web.views.icon import icon_bp +from newspipe.web.views.admin import admin_bp +from newspipe.web.views.user import user_bp, users_bp +from newspipe.web.views.bookmark import bookmark_bp, bookmarks_bp diff --git a/newspipe/web/views/admin.py b/newspipe/web/views/admin.py index fe1b389b..7df683a7 100644 --- a/newspipe/web/views/admin.py +++ b/newspipe/web/views/admin.py @@ -3,10 +3,10 @@ from flask import Blueprint, render_template, redirect, flash, url_for from flask_babel import gettext, format_timedelta from flask_login import login_required, current_user -from lib.utils import redirect_url -from web.views.common import admin_permission -from web.controllers import UserController -from web.forms import InformationMessageForm, UserForm +from newspipe.lib.utils import redirect_url +from newspipe.controllers import UserController +from newspipe.web.views.common import admin_permission +from newspipe.web.forms import InformationMessageForm, UserForm admin_bp = Blueprint("admin", __name__, url_prefix="/admin") diff --git a/newspipe/web/views/api/v2/__init__.py b/newspipe/web/views/api/v2/__init__.py index ef587e72..d367da20 100644 --- a/newspipe/web/views/api/v2/__init__.py +++ b/newspipe/web/views/api/v2/__init__.py @@ -1,3 +1,3 @@ -from web.views.api.v2 import article, feed, category +from newspipe.web.views.api.v2 import article, feed, category __all__ = ["article", "feed", "category"] diff --git a/newspipe/web/views/api/v2/article.py b/newspipe/web/views/api/v2/article.py index 8da6c6dd..1dbf23b2 100644 --- a/newspipe/web/views/api/v2/article.py +++ b/newspipe/web/views/api/v2/article.py @@ -1,12 +1,12 @@ -from conf import API_ROOT +from newspipe.bootstrap import application import dateutil.parser from datetime import datetime from flask import current_app from flask_restful import Api -from web.views.common import api_permission -from web.controllers import ArticleController -from web.views.api.v2.common import ( +from newspipe.web.views.common import api_permission +from newspipe.controllers import ArticleController +from newspipe.web.views.api.v2.common import ( PyAggAbstractResource, PyAggResourceNew, PyAggResourceExisting, @@ -49,7 +49,7 @@ class ArticlesChallenge(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json") api.add_resource(ArticleAPI, "/article/<int:obj_id>", endpoint="article.json") diff --git a/newspipe/web/views/api/v2/category.py b/newspipe/web/views/api/v2/category.py index a830624d..5f7ef354 100644 --- a/newspipe/web/views/api/v2/category.py +++ b/newspipe/web/views/api/v2/category.py @@ -1,9 +1,9 @@ -from conf import API_ROOT from flask import current_app from flask_restful import Api -from web.controllers.category import CategoryController -from web.views.api.v2.common import ( +from newspipe.bootstrap import application +from newspipe.controllers.category import CategoryController +from newspipe.web.views.api.v2.common import ( PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti, @@ -22,7 +22,7 @@ class CategoriesAPI(PyAggResourceMulti): controller_cls = CategoryController -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json") api.add_resource(CategoryAPI, "/category/<int:obj_id>", endpoint="category.json") api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json") diff --git a/newspipe/web/views/api/v2/common.py b/newspipe/web/views/api/v2/common.py index 81248422..3d90bf91 100644 --- a/newspipe/web/views/api/v2/common.py +++ b/newspipe/web/views/api/v2/common.py @@ -26,13 +26,13 @@ from flask import request from flask_restful import Resource, reqparse from flask_login import current_user -from web.views.common import ( +from newspipe.web.views.common import ( admin_permission, api_permission, login_user_bundle, jsonify, ) -from web.controllers import UserController +from newspipe.controllers import UserController logger = logging.getLogger(__name__) diff --git a/newspipe/web/views/api/v2/feed.py b/newspipe/web/views/api/v2/feed.py index 1e4fabf2..1081ed8c 100644 --- a/newspipe/web/views/api/v2/feed.py +++ b/newspipe/web/views/api/v2/feed.py @@ -1,11 +1,11 @@ -from conf import API_ROOT from flask import current_app from flask_restful import Api -from web.views.common import api_permission -from web.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT +from newspipe.bootstrap import application +from newspipe.web.views.common import api_permission +from newspipe.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from web.views.api.v2.common import ( +from newspipe.web.views.api.v2.common import ( PyAggAbstractResource, PyAggResourceNew, PyAggResourceExisting, @@ -39,7 +39,7 @@ class FetchableFeedAPI(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=API_ROOT) +api = Api(current_app, prefix=application.config['API_ROOT']) api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json") api.add_resource(FeedAPI, "/feed/<int:obj_id>", endpoint="feed.json") diff --git a/newspipe/web/views/article.py b/newspipe/web/views/article.py index c0c6f346..a49859ca 100644 --- a/newspipe/web/views/article.py +++ b/newspipe/web/views/article.py @@ -14,11 +14,11 @@ from flask_babel import gettext from flask_login import login_required, current_user -from bootstrap import db -from lib.utils import clear_string, redirect_url -from lib.data import export_json -from web.controllers import ArticleController, UserController, CategoryController -from web.lib.view_utils import etag_match +from newspipe.bootstrap import db +from newspipe.lib.utils import clear_string, redirect_url +from newspipe.lib.data import export_json +from newspipe.controllers import ArticleController, UserController, CategoryController +from newspipe.web.lib.view_utils import etag_match articles_bp = Blueprint("articles", __name__, url_prefix="/articles") article_bp = Blueprint("article", __name__, url_prefix="/article") diff --git a/newspipe/web/views/bookmark.py b/newspipe/web/views/bookmark.py index 2577f747..ea49b3c8 100644 --- a/newspipe/web/views/bookmark.py +++ b/newspipe/web/views/bookmark.py @@ -44,13 +44,12 @@ from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args from sqlalchemy import desc -import conf -from lib.utils import redirect_url -from lib.data import import_pinboard_json, export_bookmarks -from bootstrap import db -from web.forms import BookmarkForm -from web.controllers import BookmarkController, BookmarkTagController -from web.models import BookmarkTag +from newspipe.lib.utils import redirect_url +from newspipe.lib.data import import_pinboard_json, export_bookmarks +from newspipe.bootstrap import db +from newspipe.web.forms import BookmarkForm +from newspipe.controllers import BookmarkController, BookmarkTagController +from newspipe.models import BookmarkTag logger = logging.getLogger(__name__) bookmarks_bp = Blueprint("bookmarks", __name__, url_prefix="/bookmarks") diff --git a/newspipe/web/views/category.py b/newspipe/web/views/category.py index 1c897058..9f1a7e83 100644 --- a/newspipe/web/views/category.py +++ b/newspipe/web/views/category.py @@ -2,10 +2,10 @@ from flask import Blueprint, render_template, flash, redirect, url_for from flask_babel import gettext from flask_login import login_required, current_user -from web.forms import CategoryForm -from lib.utils import redirect_url -from web.lib.view_utils import etag_match -from web.controllers import ArticleController, FeedController, CategoryController +from newspipe.web.forms import CategoryForm +from newspipe.lib.utils import redirect_url +from newspipe.web.lib.view_utils import etag_match +from newspipe.controllers import ArticleController, FeedController, CategoryController categories_bp = Blueprint("categories", __name__, url_prefix="/categories") category_bp = Blueprint("category", __name__, url_prefix="/category") diff --git a/newspipe/web/views/common.py b/newspipe/web/views/common.py index c2d8e2df..8d9ecfa9 100644 --- a/newspipe/web/views/common.py +++ b/newspipe/web/views/common.py @@ -10,8 +10,8 @@ from flask_principal import ( session_identity_loader, identity_changed, ) -from web.controllers import UserController -from lib.utils import default_handler +from newspipe.controllers import UserController +from newspipe.lib.utils import default_handler admin_role = RoleNeed("admin") api_role = RoleNeed("api") diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py index 592e3cbf..0be30668 100644 --- a/newspipe/web/views/feed.py +++ b/newspipe/web/views/feed.py @@ -17,12 +17,12 @@ from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args -import conf -from lib import misc_utils, utils -from lib.feed_utils import construct_feed_from -from web.lib.view_utils import etag_match -from web.forms import AddFeedForm -from web.controllers import ( +from newspipe.bootstrap import application +from newspipe.lib import misc_utils, utils +from newspipe.lib.feed_utils import construct_feed_from +from newspipe.web.lib.view_utils import etag_match +from newspipe.web.forms import AddFeedForm +from newspipe.controllers import ( UserController, CategoryController, FeedController, @@ -179,7 +179,7 @@ def bookmarklet(): ) feed = feed_contr.create(**feed) flash(gettext("Feed was successfully created."), "success") - if feed.enabled and conf.CRAWLING_METHOD == "default": + if feed.enabled and application.confg['CRAWLING_METHOD'] == "default": misc_utils.fetch(current_user.id, feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=feed.id)) @@ -286,7 +286,7 @@ def process_form(feed_id=None): "success", ) - if conf.CRAWLING_METHOD == "default": + if application.confg['CRAWLING_METHOD'] == "default": misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), "info") @@ -335,7 +335,7 @@ def export(): if not include_private: filter["private"] = False if not include_exceeded_error_count: - filter["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filter["error_count__lt"] = application.confg['DEFAULT_MAX_ERROR'] user = UserController(current_user.id).get(id=current_user.id) feeds = FeedController(current_user.id).read(**filter) diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py index 1cfa3601..66ce87ea 100644 --- a/newspipe/web/views/home.py +++ b/newspipe/web/views/home.py @@ -7,13 +7,13 @@ from flask_login import login_required, current_user from flask_babel import gettext, get_locale from babel.dates import format_datetime, format_timedelta -import conf -from lib.utils import redirect_url -from lib import misc_utils -from web.lib.view_utils import etag_match -from web.views.common import jsonify +from newspipe.bootstrap import application +from newspipe.lib.utils import redirect_url +from newspipe.lib import misc_utils +from newspipe.web.lib.view_utils import etag_match +from newspipe.web.views.common import jsonify -from web.controllers import FeedController, ArticleController, CategoryController +from newspipe.controllers import FeedController, ArticleController, CategoryController localize = pytz.utc.localize logger = logging.getLogger(__name__) @@ -181,7 +181,7 @@ def fetch(feed_id=None): Triggers the download of news. News are downloaded in a separated process. """ - if conf.CRAWLING_METHOD == "default" and current_user.is_admin: + if application.config['CRAWLING_METHOD'] == "default" and current_user.is_admin: misc_utils.fetch(current_user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: diff --git a/newspipe/web/views/icon.py b/newspipe/web/views/icon.py index e1de6402..4cdcd4b0 100644 --- a/newspipe/web/views/icon.py +++ b/newspipe/web/views/icon.py @@ -1,7 +1,8 @@ import base64 from flask import Blueprint, Response, request -from web.controllers import IconController -from web.lib.view_utils import etag_match + +from newspipe.controllers import IconController +from newspipe.web.lib.view_utils import etag_match icon_bp = Blueprint("icon", __name__, url_prefix="/icon") diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py index 809825d3..9bdb89cb 100644 --- a/newspipe/web/views/session_mgmt.py +++ b/newspipe/web/views/session_mgmt.py @@ -24,11 +24,11 @@ from flask_principal import ( session_identity_loader, ) -import conf -from web.views.common import admin_role, api_role, login_user_bundle -from web.controllers import UserController -from web.forms import SignupForm, SigninForm -from notifications import notifications +from newspipe.bootstrap import application +from newspipe.web.views.common import admin_role, api_role, login_user_bundle +from newspipe.controllers import UserController +from newspipe.web.forms import SignupForm, SigninForm +from newspipe.notifications import notifications Principal(current_app) # Create a permission with a single Need, in this case a RoleNeed. @@ -99,7 +99,7 @@ def logout(): @current_app.route("/signup", methods=["GET", "POST"]) def signup(): - if not conf.SELF_REGISTRATION: + if not application.config['SELF_REGISTRATION']: flash(gettext("Self-registration is disabled."), "warning") return redirect(url_for("home")) if current_user.is_authenticated: diff --git a/newspipe/web/views/user.py b/newspipe/web/views/user.py index 10974947..39eefb4c 100644 --- a/newspipe/web/views/user.py +++ b/newspipe/web/views/user.py @@ -6,12 +6,12 @@ from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args -import conf -from notifications import notifications -from lib import misc_utils -from lib.data import import_opml, import_json -from web.lib.user_utils import confirm_token -from web.controllers import ( +from newspipe.bootstrap import application +from newspipe.notifications import notifications +from newspipe.lib import misc_utils +from newspipe.lib.data import import_opml, import_json +from newspipe.web.lib.user_utils import confirm_token +from newspipe.controllers import ( UserController, FeedController, ArticleController, @@ -19,7 +19,7 @@ from web.controllers import ( BookmarkController, ) -from web.forms import ProfileForm +from newspipe.web.forms import ProfileForm users_bp = Blueprint("users", __name__, url_prefix="/users") user_bp = Blueprint("user", __name__, url_prefix="/user") @@ -115,7 +115,7 @@ def management(): else: try: nb = import_opml(current_user.nickname, data.read()) - if conf.CRAWLING_METHOD == "classic": + if application.config['CRAWLING_METHOD'] == "classic": misc_utils.fetch(current_user.id, None) flash(str(nb) + " " + gettext("feeds imported."), "success") flash(gettext("Downloading articles..."), "info") diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py index e08270cd..6aa6ce54 100644 --- a/newspipe/web/views/views.py +++ b/newspipe/web/views/views.py @@ -6,11 +6,10 @@ from flask import request, render_template, flash, url_for, redirect, current_ap from flask_babel import gettext from sqlalchemy import desc -import conf -from web import __version__ -from conf import API_ROOT, ADMIN_EMAIL -from web.controllers import FeedController, UserController -from web.lib.view_utils import etag_match +from newspipe.bootstrap import application +from newspipe.web import __version__ +from newspipe.controllers import FeedController, UserController +from newspipe.web.lib.view_utils import etag_match logger = logging.getLogger(__name__) @@ -25,7 +24,7 @@ def authentication_required(error): @current_app.errorhandler(403) def authentication_failed(error): - if API_ROOT in request.url: + if application.conf['API_ROOT'] in request.url: return error flash(gettext("Forbidden."), "danger") return redirect(url_for("login")) @@ -71,7 +70,7 @@ def popular(): filters = {} filters["created_date__gt"] = not_added_before filters["private"] = False - filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] feeds = FeedController().count_by_link(**filters) sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True) return render_template("popular.html", popular=sorted_feeds) @@ -80,7 +79,7 @@ def popular(): @current_app.route("/about", methods=["GET"]) @etag_match def about(): - return render_template("about.html", contact=ADMIN_EMAIL) + return render_template("about.html", contact=application.config['ADMIN_EMAIL']) @current_app.route("/about/more", methods=["GET"]) @@ -102,7 +101,7 @@ def about_more(): "about_more.html", newspipe_version=newspipe_version, version_url=version_url, - registration=[conf.SELF_REGISTRATION and "Open" or "Closed"][0], + registration=[application.config['SELF_REGISTRATION'] and "Open" or "Closed"][0], python_version="{}.{}.{}".format(*sys.version_info[:3]), nb_users=UserController().read().count(), ) diff --git a/package-lock.json b/package-lock.json index f17279a2..106e200a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "resolved": "https://registry.npmjs.org/datatables/-/datatables-1.10.18.tgz", "integrity": "sha512-ntatMgS9NN6UMpwbmO+QkYJuKlVeMA2Mi0Gu/QxyIh+dW7ZjLSDhPT2tWlzjpIWEkDYgieDzS9Nu7bdQCW0sbQ==", "requires": { - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "datatables.net": { @@ -22,7 +22,7 @@ "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-1.10.20.tgz", "integrity": "sha512-4E4S7tTU607N3h0fZPkGmAtr9mwy462u+VJ6gxYZ8MxcRIjZqHy3Dv1GNry7i3zQCktTdWbULVKBbkAJkuHEnQ==", "requires": { - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "datatables.net-bs4": { @@ -31,7 +31,7 @@ "integrity": "sha512-kQmMUMsHMOlAW96ztdoFqjSbLnlGZQ63iIM82kHbmldsfYdzuyhbb4hTx6YNBi481WCO3iPSvI6YodNec46ZAw==", "requires": { "datatables.net": "1.10.20", - "jquery": ">=1.7" + "jquery": "3.4.1" } }, "fork-awesome": { diff --git a/package.json b/package.json index 099771b0..e89900cd 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,6 @@ "yarn": ">= 1.0.0" }, "scripts": { - "postinstall": "cd newspipe/web/static/ ; ln -sf ../../../node_modules npm_components" + "postinstall": "cd newspipe/static/ ; ln -sf ../../node_modules npm_components" } } diff --git a/newspipe/runserver.py b/runserver.py index 8880d5b9..20c712a4 100755 --- a/newspipe/runserver.py +++ b/runserver.py @@ -19,7 +19,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import calendar -from bootstrap import conf, application, populate_g +from newspipe.bootstrap import application from flask_babel import Babel, format_datetime @@ -33,18 +33,18 @@ def month_name(month_number): application.jinja_env.filters["month_name"] = month_name application.jinja_env.filters["datetime"] = format_datetime -application.jinja_env.globals["conf"] = conf +# inject application in Jinja env +application.jinja_env.globals["application"] = application # Views from flask_restful import Api from flask import g with application.app_context(): - populate_g() g.api = Api(application, prefix="/api/v2.0") g.babel = babel - from web import views + from newspipe.web import views application.register_blueprint(views.articles_bp) application.register_blueprint(views.article_bp) @@ -62,5 +62,7 @@ with application.app_context(): if __name__ == "__main__": application.run( - host=conf.WEBSERVER_HOST, port=conf.WEBSERVER_PORT, debug=conf.WEBSERVER_DEBUG + host=application.config['HOST'], + port=application.config['PORT'], + debug=application.config['DEBUG'] ) |