diff options
Diffstat (limited to 'newspipe')
-rw-r--r-- | newspipe/bootstrap.py | 2 | ||||
-rw-r--r-- | newspipe/controllers/feed.py | 2 | ||||
-rw-r--r-- | newspipe/crawler/default_crawler.py | 10 | ||||
-rw-r--r-- | newspipe/lib/article_utils.py | 7 | ||||
-rw-r--r-- | newspipe/lib/feed_utils.py | 5 | ||||
-rwxr-xr-x | newspipe/lib/misc_utils.py | 4 | ||||
-rw-r--r-- | newspipe/lib/utils.py | 4 | ||||
-rw-r--r-- | newspipe/notifications/emails.py | 20 | ||||
-rw-r--r-- | newspipe/notifications/notifications.py | 8 | ||||
-rw-r--r-- | newspipe/web/__init__.py | 8 | ||||
-rw-r--r-- | newspipe/web/lib/user_utils.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/article.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/category.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/api/v2/feed.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/feed.py | 6 | ||||
-rw-r--r-- | newspipe/web/views/home.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/session_mgmt.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/user.py | 2 | ||||
-rw-r--r-- | newspipe/web/views/views.py | 14 |
19 files changed, 61 insertions, 43 deletions
diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py index 1fd1ab04..0d140fae 100644 --- a/newspipe/bootstrap.py +++ b/newspipe/bootstrap.py @@ -62,7 +62,7 @@ else: # application.config["SERVER_NAME"] = domain # application.config["PREFERRED_URL_SCHEME"] = scheme -set_logging(application.config['LOG_PATH']) +set_logging(application.config["LOG_PATH"]) db = SQLAlchemy(application) diff --git a/newspipe/controllers/feed.py b/newspipe/controllers/feed.py index 798efa6e..73f852a8 100644 --- a/newspipe/controllers/feed.py +++ b/newspipe/controllers/feed.py @@ -10,7 +10,7 @@ from newspipe.lib.utils import clear_string logger = logging.getLogger(__name__) DEFAULT_LIMIT = 5 -DEFAULT_MAX_ERROR = application.config['DEFAULT_MAX_ERROR'] +DEFAULT_MAX_ERROR = application.config["DEFAULT_MAX_ERROR"] class FeedController(AbstractController): diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py index 9296e5e4..c12aad0d 100644 --- a/newspipe/crawler/default_crawler.py +++ b/newspipe/crawler/default_crawler.py @@ -40,7 +40,11 @@ from newspipe.models import User from newspipe.controllers import FeedController, ArticleController from newspipe.lib.utils import newspipe_get from newspipe.lib.feed_utils import construct_feed_from, is_parsing_ok -from newspipe.lib.article_utils import construct_article, extract_id, get_article_content +from newspipe.lib.article_utils import ( + construct_article, + extract_id, + get_article_content, +) logger = logging.getLogger(__name__) @@ -163,9 +167,9 @@ async def retrieve_feed(queue, users, feed_id=None): if feed_id is not None: filters["id"] = feed_id filters["enabled"] = True - filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] + filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"] filters["last_retrieved__lt"] = datetime.now() - timedelta( - minutes=application.config['FEED_REFRESH_INTERVAL'] + minutes=application.config["FEED_REFRESH_INTERVAL"] ) feeds = FeedController().read(**filters).all() diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py index 77cea397..5152f008 100644 --- a/newspipe/lib/article_utils.py +++ b/newspipe/lib/article_utils.py @@ -77,7 +77,12 @@ def get_article_content(entry): async def get_article_details(entry, fetch=True): article_link = entry.get("link") article_title = html.unescape(entry.get("title", "")) - if fetch and application.config['CRAWLER_RESOLV'] and article_link or not article_title: + if ( + fetch + and application.config["CRAWLER_RESOLV"] + and article_link + or not article_title + ): try: # resolves URL behind proxies (like feedproxy.google.com) response = await newspipe_get(article_link, timeout=5) diff --git a/newspipe/lib/feed_utils.py b/newspipe/lib/feed_utils.py index a4c6c275..55cb6205 100644 --- a/newspipe/lib/feed_utils.py +++ b/newspipe/lib/feed_utils.py @@ -39,7 +39,10 @@ def escape_keys(*keys): @escape_keys("title", "description") def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): - requests_kwargs = {"headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, "verify": False} + requests_kwargs = { + "headers": {"User-Agent": application.config["CRAWLER_USER_AGENT"]}, + "verify": False, + } if url is None and fp_parsed is not None: url = fp_parsed.get("url") if url is not None and fp_parsed is None: diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py index b1ccd075..cfe2b093 100755 --- a/newspipe/lib/misc_utils.py +++ b/newspipe/lib/misc_utils.py @@ -101,7 +101,7 @@ def fetch(id, feed_id=None): """ cmd = [ sys.executable, - application.config['BASE_DIR'] + "/manager.py", + application.config["BASE_DIR"] + "/manager.py", "fetch_asyncio", "--user_id=" + str(id), ] @@ -154,7 +154,7 @@ def load_stop_words(): Load the stop words and return them in a list. """ stop_words_lists = glob.glob( - os.path.join(application.config['BASE_DIR'], "web/var/stop_words/*.txt") + os.path.join(application.config["BASE_DIR"], "web/var/stop_words/*.txt") ) stop_words = [] diff --git a/newspipe/lib/utils.py b/newspipe/lib/utils.py index 958c9f23..67ced60d 100644 --- a/newspipe/lib/utils.py +++ b/newspipe/lib/utils.py @@ -93,8 +93,8 @@ async def newspipe_get(url, **kwargs): request_kwargs = { "verify": False, "allow_redirects": True, - "timeout": application.config['CRAWLER_TIMEOUT'], - "headers": {"User-Agent": application.config['CRAWLER_USER_AGENT']}, + "timeout": application.config["CRAWLER_TIMEOUT"], + "headers": {"User-Agent": application.config["CRAWLER_USER_AGENT"]}, } request_kwargs.update(kwargs) return requests.get(url, **request_kwargs) diff --git a/newspipe/notifications/emails.py b/newspipe/notifications/emails.py index 9738ae95..8093b655 100644 --- a/newspipe/notifications/emails.py +++ b/newspipe/notifications/emails.py @@ -33,8 +33,11 @@ logger = logging.getLogger(__name__) @async_maker def send_async_email(mfrom, mto, msg): try: - s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) - s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) + s = smtplib.SMTP(application.config["NOTIFICATION_HOST"]) + s.login( + application.config["NOTIFICATION_USERNAME"], + application.config["NOTIFICATION_PASSWORD"], + ) except Exception: logger.exception("send_async_email raised:") else: @@ -56,7 +59,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): # Create message container - the correct MIME type is multipart/alternative. msg = MIMEMultipart("alternative") msg["Subject"] = subject - msg["From"] = application.config['NOTIFICATION_EMAIL'] + msg["From"] = application.config["NOTIFICATION_EMAIL"] msg["To"] = to msg["BCC"] = bcc @@ -71,12 +74,17 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): msg.attach(part2) try: - s = smtplib.SMTP(application.config['NOTIFICATION_HOST']) - s.login(application.config['NOTIFICATION_USERNAME'], application.config['NOTIFICATION_PASSWORD']) + s = smtplib.SMTP(application.config["NOTIFICATION_HOST"]) + s.login( + application.config["NOTIFICATION_USERNAME"], + application.config["NOTIFICATION_PASSWORD"], + ) except Exception: logger.exception("send_smtp raised:") else: s.sendmail( - application.config['NOTIFICATION_EMAIL'], msg["To"] + ", " + msg["BCC"], msg.as_string() + application.config["NOTIFICATION_EMAIL"], + msg["To"] + ", " + msg["BCC"], + msg.as_string(), ) s.quit() diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py index 71ab0cf8..f5421e59 100644 --- a/newspipe/notifications/notifications.py +++ b/newspipe/notifications/notifications.py @@ -33,20 +33,20 @@ def new_account_notification(user, email): """ token = generate_confirmation_token(user.nickname) expire_time = datetime.datetime.now() + datetime.timedelta( - seconds=application.config['TOKEN_VALIDITY_PERIOD'] + seconds=application.config["TOKEN_VALIDITY_PERIOD"] ) plaintext = render_template( "emails/account_activation.txt", user=user, - platform_url=application.config['PLATFORM_URL'], + platform_url=application.config["PLATFORM_URL"], token=token, expire_time=expire_time, ) emails.send( to=email, - bcc=application.config['NOTIFICATION_EMAIL'], + bcc=application.config["NOTIFICATION_EMAIL"], subject="[Newspipe] Account creation", plaintext=plaintext, ) @@ -59,7 +59,7 @@ def new_password_notification(user, password): plaintext = render_template("emails/new_password.txt", user=user, password=password) emails.send( to=user.email, - bcc=application.config['NOTIFICATION_EMAIL'], + bcc=application.config["NOTIFICATION_EMAIL"], subject="[Newspipe] New password", plaintext=plaintext, ) diff --git a/newspipe/web/__init__.py b/newspipe/web/__init__.py index 16d84cc5..9cd78b88 100644 --- a/newspipe/web/__init__.py +++ b/newspipe/web/__init__.py @@ -12,11 +12,9 @@ BASE_DIR = os.path.abspath(os.path.dirname(__file__)) __version__ = ( os.environ.get("PKGVER") - or subprocess.run(["git", - "-C", - BASE_DIR, - "describe", - "--tags"], stdout=subprocess.PIPE) + or subprocess.run( + ["git", "-C", BASE_DIR, "describe", "--tags"], stdout=subprocess.PIPE + ) .stdout.decode() .strip() ) diff --git a/newspipe/web/lib/user_utils.py b/newspipe/web/lib/user_utils.py index 95b436a9..22cf3b6a 100644 --- a/newspipe/web/lib/user_utils.py +++ b/newspipe/web/lib/user_utils.py @@ -14,7 +14,7 @@ def confirm_token(token): nickname = serializer.loads( token, salt=application.config["SECURITY_PASSWORD_SALT"], - max_age=application.config['TOKEN_VALIDITY_PERIOD'], + max_age=application.config["TOKEN_VALIDITY_PERIOD"], ) except: return False diff --git a/newspipe/web/views/api/v2/article.py b/newspipe/web/views/api/v2/article.py index 1dbf23b2..b35d2635 100644 --- a/newspipe/web/views/api/v2/article.py +++ b/newspipe/web/views/api/v2/article.py @@ -49,7 +49,7 @@ class ArticlesChallenge(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=application.config['API_ROOT']) +api = Api(current_app, prefix=application.config["API_ROOT"]) api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json") api.add_resource(ArticleAPI, "/article/<int:obj_id>", endpoint="article.json") diff --git a/newspipe/web/views/api/v2/category.py b/newspipe/web/views/api/v2/category.py index 5f7ef354..697a283f 100644 --- a/newspipe/web/views/api/v2/category.py +++ b/newspipe/web/views/api/v2/category.py @@ -22,7 +22,7 @@ class CategoriesAPI(PyAggResourceMulti): controller_cls = CategoryController -api = Api(current_app, prefix=application.config['API_ROOT']) +api = Api(current_app, prefix=application.config["API_ROOT"]) api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json") api.add_resource(CategoryAPI, "/category/<int:obj_id>", endpoint="category.json") api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json") diff --git a/newspipe/web/views/api/v2/feed.py b/newspipe/web/views/api/v2/feed.py index 1081ed8c..eaccd24a 100644 --- a/newspipe/web/views/api/v2/feed.py +++ b/newspipe/web/views/api/v2/feed.py @@ -39,7 +39,7 @@ class FetchableFeedAPI(PyAggAbstractResource): return result or None, 200 if result else 204 -api = Api(current_app, prefix=application.config['API_ROOT']) +api = Api(current_app, prefix=application.config["API_ROOT"]) api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json") api.add_resource(FeedAPI, "/feed/<int:obj_id>", endpoint="feed.json") diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py index 0be30668..ec69e301 100644 --- a/newspipe/web/views/feed.py +++ b/newspipe/web/views/feed.py @@ -179,7 +179,7 @@ def bookmarklet(): ) feed = feed_contr.create(**feed) flash(gettext("Feed was successfully created."), "success") - if feed.enabled and application.confg['CRAWLING_METHOD'] == "default": + if feed.enabled and application.confg["CRAWLING_METHOD"] == "default": misc_utils.fetch(current_user.id, feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=feed.id)) @@ -286,7 +286,7 @@ def process_form(feed_id=None): "success", ) - if application.confg['CRAWLING_METHOD'] == "default": + if application.confg["CRAWLING_METHOD"] == "default": misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), "info") @@ -335,7 +335,7 @@ def export(): if not include_private: filter["private"] = False if not include_exceeded_error_count: - filter["error_count__lt"] = application.confg['DEFAULT_MAX_ERROR'] + filter["error_count__lt"] = application.confg["DEFAULT_MAX_ERROR"] user = UserController(current_user.id).get(id=current_user.id) feeds = FeedController(current_user.id).read(**filter) diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py index 66ce87ea..0fe09b17 100644 --- a/newspipe/web/views/home.py +++ b/newspipe/web/views/home.py @@ -181,7 +181,7 @@ def fetch(feed_id=None): Triggers the download of news. News are downloaded in a separated process. """ - if application.config['CRAWLING_METHOD'] == "default" and current_user.is_admin: + if application.config["CRAWLING_METHOD"] == "default" and current_user.is_admin: misc_utils.fetch(current_user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py index 9bdb89cb..80f505c6 100644 --- a/newspipe/web/views/session_mgmt.py +++ b/newspipe/web/views/session_mgmt.py @@ -99,7 +99,7 @@ def logout(): @current_app.route("/signup", methods=["GET", "POST"]) def signup(): - if not application.config['SELF_REGISTRATION']: + if not application.config["SELF_REGISTRATION"]: flash(gettext("Self-registration is disabled."), "warning") return redirect(url_for("home")) if current_user.is_authenticated: diff --git a/newspipe/web/views/user.py b/newspipe/web/views/user.py index 39eefb4c..78002660 100644 --- a/newspipe/web/views/user.py +++ b/newspipe/web/views/user.py @@ -115,7 +115,7 @@ def management(): else: try: nb = import_opml(current_user.nickname, data.read()) - if application.config['CRAWLING_METHOD'] == "classic": + if application.config["CRAWLING_METHOD"] == "classic": misc_utils.fetch(current_user.id, None) flash(str(nb) + " " + gettext("feeds imported."), "success") flash(gettext("Downloading articles..."), "info") diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py index 6aa6ce54..e3a5091f 100644 --- a/newspipe/web/views/views.py +++ b/newspipe/web/views/views.py @@ -24,7 +24,7 @@ def authentication_required(error): @current_app.errorhandler(403) def authentication_failed(error): - if application.conf['API_ROOT'] in request.url: + if application.conf["API_ROOT"] in request.url: return error flash(gettext("Forbidden."), "danger") return redirect(url_for("login")) @@ -70,7 +70,7 @@ def popular(): filters = {} filters["created_date__gt"] = not_added_before filters["private"] = False - filters["error_count__lt"] = application.config['DEFAULT_MAX_ERROR'] + filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"] feeds = FeedController().count_by_link(**filters) sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True) return render_template("popular.html", popular=sorted_feeds) @@ -79,7 +79,7 @@ def popular(): @current_app.route("/about", methods=["GET"]) @etag_match def about(): - return render_template("about.html", contact=application.config['ADMIN_EMAIL']) + return render_template("about.html", contact=application.config["ADMIN_EMAIL"]) @current_app.route("/about/more", methods=["GET"]) @@ -88,9 +88,7 @@ def about_more(): version = __version__.split("-") if len(version) == 1: newspipe_version = version[0] - version_url = "https://git.sr.ht/~cedric/newspipe/refs/{}".format( - version[0] - ) + version_url = "https://git.sr.ht/~cedric/newspipe/refs/{}".format(version[0]) else: newspipe_version = "{} - {}".format(version[0], version[2][1:]) version_url = "https://git.sr.ht/~cedric/newspipe/commit/{}".format( @@ -101,7 +99,9 @@ def about_more(): "about_more.html", newspipe_version=newspipe_version, version_url=version_url, - registration=[application.config['SELF_REGISTRATION'] and "Open" or "Closed"][0], + registration=[application.config["SELF_REGISTRATION"] and "Open" or "Closed"][ + 0 + ], python_version="{}.{}.{}".format(*sys.version_info[:3]), nb_users=UserController().read().count(), ) |