diff options
54 files changed, 1947 insertions, 1408 deletions
diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py index 004bebf0..01e38fbf 100644 --- a/newspipe/bootstrap.py +++ b/newspipe/bootstrap.py @@ -10,16 +10,27 @@ import flask_restless from urllib.parse import urlsplit -def set_logging(log_path=None, log_level=logging.INFO, modules=(), - log_format='%(asctime)s %(levelname)s %(message)s'): +def set_logging( + log_path=None, + log_level=logging.INFO, + modules=(), + log_format="%(asctime)s %(levelname)s %(message)s", +): if not modules: - modules = ('root', 'bootstrap', 'runserver', - 'web', 'crawler.default_crawler', 'manager', 'plugins') + modules = ( + "root", + "bootstrap", + "runserver", + "web", + "crawler.default_crawler", + "manager", + "plugins", + ) if log_path: if not os.path.exists(os.path.dirname(log_path)): os.makedirs(os.path.dirname(log_path)) if not os.path.exists(log_path): - open(log_path, 'w').close() + open(log_path, "w").close() handler = logging.FileHandler(log_path) else: handler = logging.StreamHandler() @@ -32,39 +43,40 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(), handler.setLevel(log_level) logger.setLevel(log_level) + from flask import Flask from flask_sqlalchemy import SQLAlchemy # Create Flask application -application = Flask('web') -if os.environ.get('Newspipe_TESTING', False) == 'true': +application = Flask("web") +if os.environ.get("Newspipe_TESTING", False) == "true": application.debug = logging.DEBUG - application.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' - application.config['TESTING'] = True + application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" + application.config["TESTING"] = True else: application.debug = conf.LOG_LEVEL <= logging.DEBUG - application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - application.config['SQLALCHEMY_DATABASE_URI'] \ - = conf.SQLALCHEMY_DATABASE_URI - if 'postgres' in conf.SQLALCHEMY_DATABASE_URI: - application.config['SQLALCHEMY_POOL_SIZE'] = 15 - application.config['SQLALCHEMY_MAX_OVERFLOW'] = 0 + application.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + application.config["SQLALCHEMY_DATABASE_URI"] = conf.SQLALCHEMY_DATABASE_URI + if "postgres" in conf.SQLALCHEMY_DATABASE_URI: + application.config["SQLALCHEMY_POOL_SIZE"] = 15 + application.config["SQLALCHEMY_MAX_OVERFLOW"] = 0 scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL) -application.config['SERVER_NAME'] = domain -application.config['PREFERRED_URL_SCHEME'] = scheme +application.config["SERVER_NAME"] = domain +application.config["PREFERRED_URL_SCHEME"] = scheme set_logging(conf.LOG_PATH, log_level=conf.LOG_LEVEL) # Create secrey key so we can use sessions -application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) -if not application.config['SECRET_KEY']: - application.config['SECRET_KEY'] = os.urandom(12) +application.config["SECRET_KEY"] = getattr(conf, "WEBSERVER_SECRET", None) +if not application.config["SECRET_KEY"]: + application.config["SECRET_KEY"] = os.urandom(12) -application.config['SECURITY_PASSWORD_SALT'] = getattr(conf, - 'SECURITY_PASSWORD_SALT', None) -if not application.config['SECURITY_PASSWORD_SALT']: - application.config['SECURITY_PASSWORD_SALT'] = os.urandom(12) +application.config["SECURITY_PASSWORD_SALT"] = getattr( + conf, "SECURITY_PASSWORD_SALT", None +) +if not application.config["SECURITY_PASSWORD_SALT"]: + application.config["SECURITY_PASSWORD_SALT"] = os.urandom(12) db = SQLAlchemy(application) @@ -74,5 +86,6 @@ manager = flask_restless.APIManager(application, flask_sqlalchemy_db=db) def populate_g(): from flask import g + g.db = db g.app = application diff --git a/newspipe/conf.py b/newspipe/conf.py index fcdaef75..bb51e563 100644 --- a/newspipe/conf.py +++ b/newspipe/conf.py @@ -10,41 +10,36 @@ import logging BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PATH = os.path.abspath(".") -API_ROOT = '/api/v2.0' +API_ROOT = "/api/v2.0" # available languages -LANGUAGES = { - 'en': 'English', - 'fr': 'French' -} +LANGUAGES = {"en": "English", "fr": "French"} -TIME_ZONE = { - "en": "US/Eastern", - "fr": "Europe/Paris" -} +TIME_ZONE = {"en": "US/Eastern", "fr": "Europe/Paris"} -DEFAULTS = {"platform_url": "https://www.newspipe.org/", - "self_registration": "false", - "cdn_address": "", - "admin_email": "info@newspipe.org", - "token_validity_period": "3600", - "default_max_error": "3", - "log_path": "newspipe.log", - "log_level": "info", - "secret_key": "", - "security_password_salt": "", - "enabled": "false", - "notification_email": "info@newspipe.org", - "tls": "false", - "ssl": "true", - "host": "0.0.0.0", - "port": "5000", - "crawling_method": "default", - "crawler_user_agent": "Newspipe (https://github.com/newspipe)", - "crawler_timeout": "30", - "crawler_resolv": "false", - "feed_refresh_interval": "120" - } +DEFAULTS = { + "platform_url": "https://www.newspipe.org/", + "self_registration": "false", + "cdn_address": "", + "admin_email": "info@newspipe.org", + "token_validity_period": "3600", + "default_max_error": "3", + "log_path": "newspipe.log", + "log_level": "info", + "secret_key": "", + "security_password_salt": "", + "enabled": "false", + "notification_email": "info@newspipe.org", + "tls": "false", + "ssl": "true", + "host": "0.0.0.0", + "port": "5000", + "crawling_method": "default", + "crawler_user_agent": "Newspipe (https://github.com/newspipe)", + "crawler_timeout": "30", + "crawler_resolv": "false", + "feed_refresh_interval": "120", +} # load the configuration @@ -52,51 +47,53 @@ config = confparser.SafeConfigParser(defaults=DEFAULTS) config.read(os.path.join(BASE_DIR, "conf/conf.cfg")) -WEBSERVER_HOST = config.get('webserver', 'host') -WEBSERVER_PORT = config.getint('webserver', 'port') -WEBSERVER_SECRET = config.get('webserver', 'secret_key') -WEBSERVER_DEBUG = config.getboolean('webserver', 'debug') +WEBSERVER_HOST = config.get("webserver", "host") +WEBSERVER_PORT = config.getint("webserver", "port") +WEBSERVER_SECRET = config.get("webserver", "secret_key") +WEBSERVER_DEBUG = config.getboolean("webserver", "debug") -CDN_ADDRESS = config.get('cdn', 'cdn_address') +CDN_ADDRESS = config.get("cdn", "cdn_address") try: - PLATFORM_URL = config.get('misc', 'platform_url') + PLATFORM_URL = config.get("misc", "platform_url") except: PLATFORM_URL = "https://www.newspipe.org/" -ADMIN_EMAIL = config.get('misc', 'admin_email') -SELF_REGISTRATION = config.getboolean('misc', 'self_registration') -SECURITY_PASSWORD_SALT = config.get('misc', 'security_password_salt') +ADMIN_EMAIL = config.get("misc", "admin_email") +SELF_REGISTRATION = config.getboolean("misc", "self_registration") +SECURITY_PASSWORD_SALT = config.get("misc", "security_password_salt") try: - TOKEN_VALIDITY_PERIOD = config.getint('misc', 'token_validity_period') + TOKEN_VALIDITY_PERIOD = config.getint("misc", "token_validity_period") except: - TOKEN_VALIDITY_PERIOD = int(config.get('misc', 'token_validity_period')) -LOG_PATH = os.path.abspath(config.get('misc', 'log_path')) -LOG_LEVEL = {'debug': logging.DEBUG, - 'info': logging.INFO, - 'warn': logging.WARN, - 'error': logging.ERROR, - 'fatal': logging.FATAL}[config.get('misc', 'log_level')] + TOKEN_VALIDITY_PERIOD = int(config.get("misc", "token_validity_period")) +LOG_PATH = os.path.abspath(config.get("misc", "log_path")) +LOG_LEVEL = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warn": logging.WARN, + "error": logging.ERROR, + "fatal": logging.FATAL, +}[config.get("misc", "log_level")] -SQLALCHEMY_DATABASE_URI = config.get('database', 'database_url') +SQLALCHEMY_DATABASE_URI = config.get("database", "database_url") -CRAWLING_METHOD = config.get('crawler', 'crawling_method') -CRAWLER_USER_AGENT = config.get('crawler', 'user_agent') -DEFAULT_MAX_ERROR = config.getint('crawler', 'default_max_error') +CRAWLING_METHOD = config.get("crawler", "crawling_method") +CRAWLER_USER_AGENT = config.get("crawler", "user_agent") +DEFAULT_MAX_ERROR = config.getint("crawler", "default_max_error") ERROR_THRESHOLD = int(DEFAULT_MAX_ERROR / 2) -CRAWLER_TIMEOUT = config.get('crawler', 'timeout') -CRAWLER_RESOLV = config.getboolean('crawler', 'resolv') +CRAWLER_TIMEOUT = config.get("crawler", "timeout") +CRAWLER_RESOLV = config.getboolean("crawler", "resolv") try: - FEED_REFRESH_INTERVAL = config.getint('crawler', 'feed_refresh_interval') + FEED_REFRESH_INTERVAL = config.getint("crawler", "feed_refresh_interval") except: - FEED_REFRESH_INTERVAL = int(config.get('crawler', 'feed_refresh_interval')) + FEED_REFRESH_INTERVAL = int(config.get("crawler", "feed_refresh_interval")) -NOTIFICATION_EMAIL = config.get('notification', 'notification_email') -NOTIFICATION_HOST = config.get('notification', 'host') -NOTIFICATION_PORT = config.getint('notification', 'port') -NOTIFICATION_TLS = config.getboolean('notification', 'tls') -NOTIFICATION_SSL = config.getboolean('notification', 'ssl') -NOTIFICATION_USERNAME = config.get('notification', 'username') -NOTIFICATION_PASSWORD = config.get('notification', 'password') +NOTIFICATION_EMAIL = config.get("notification", "notification_email") +NOTIFICATION_HOST = config.get("notification", "host") +NOTIFICATION_PORT = config.getint("notification", "port") +NOTIFICATION_TLS = config.getboolean("notification", "tls") +NOTIFICATION_SSL = config.getboolean("notification", "ssl") +NOTIFICATION_USERNAME = config.get("notification", "username") +NOTIFICATION_PASSWORD = config.get("notification", "password") CSRF_ENABLED = True # slow database query threshold (in seconds) diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py index 79a746b5..0ad9f4fc 100644 --- a/newspipe/crawler/default_crawler.py +++ b/newspipe/crawler/default_crawler.py @@ -40,8 +40,7 @@ from web.models import User from web.controllers import FeedController, ArticleController from lib.utils import jarr_get from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.article_utils import construct_article, extract_id, \ - get_article_content +from lib.article_utils import construct_article, extract_id, get_article_content logger = logging.getLogger(__name__) @@ -57,12 +56,12 @@ async def parse_feed(user, feed): up_feed = {} articles = [] resp = None - #with (await sem): + # with (await sem): try: - logger.info('Retrieving feed {}'.format(feed.link)) + logger.info("Retrieving feed {}".format(feed.link)) resp = await jarr_get(feed.link, timeout=5) except Exception as e: - logger.info('Problem when reading feed {}'.format(feed.link)) + logger.info("Problem when reading feed {}".format(feed.link)) return finally: if None is resp: @@ -71,38 +70,38 @@ async def parse_feed(user, feed): content = io.BytesIO(resp.content) parsed_feed = feedparser.parse(content) except Exception as e: - up_feed['last_error'] = str(e) - up_feed['error_count'] = feed.error_count + 1 + up_feed["last_error"] = str(e) + up_feed["error_count"] = feed.error_count + 1 logger.exception("error when parsing feed: " + str(e)) finally: - up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal()) + up_feed["last_retrieved"] = datetime.now(dateutil.tz.tzlocal()) if parsed_feed is None: try: - FeedController().update({'id': feed.id}, up_feed) + FeedController().update({"id": feed.id}, up_feed) except Exception as e: - logger.exception('something bad here: ' + str(e)) + logger.exception("something bad here: " + str(e)) return if not is_parsing_ok(parsed_feed): - up_feed['last_error'] = str(parsed_feed['bozo_exception']) - up_feed['error_count'] = feed.error_count + 1 - FeedController().update({'id': feed.id}, up_feed) + up_feed["last_error"] = str(parsed_feed["bozo_exception"]) + up_feed["error_count"] = feed.error_count + 1 + FeedController().update({"id": feed.id}, up_feed) return - if parsed_feed['entries'] != []: - articles = parsed_feed['entries'] + if parsed_feed["entries"] != []: + articles = parsed_feed["entries"] - up_feed['error_count'] = 0 - up_feed['last_error'] = "" + up_feed["error_count"] = 0 + up_feed["last_error"] = "" # Feed information try: construct_feed_from(feed.link, parsed_feed).update(up_feed) except: - logger.exception('error when constructing feed: {}'.format(feed.link)) - if feed.title and 'title' in up_feed: + logger.exception("error when constructing feed: {}".format(feed.link)) + if feed.title and "title" in up_feed: # do not override the title set by the user - del up_feed['title'] - FeedController().update({'id': feed.id}, up_feed) + del up_feed["title"] + FeedController().update({"id": feed.id}, up_feed) return articles @@ -116,19 +115,18 @@ async def insert_articles(queue, nḅ_producers=1): if item is None: nb_producers_done += 1 if nb_producers_done == nḅ_producers: - print('All producers done.') - print('Process finished.') + print("All producers done.") + print("Process finished.") break continue user, feed, articles = item - if None is articles: - logger.info('None') + logger.info("None") articles = [] - logger.info('Inserting articles for {}'.format(feed.link)) + logger.info("Inserting articles for {}".format(feed.link)) art_contr = ArticleController(user.id) for article in articles: @@ -136,9 +134,8 @@ async def insert_articles(queue, nḅ_producers=1): try: existing_article_req = art_contr.read( - user_id=user.id, - feed_id=feed.id, - entry_id=extract_id(article)) + user_id=user.id, feed_id=feed.id, entry_id=extract_id(article) + ) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue @@ -149,9 +146,9 @@ async def insert_articles(queue, nḅ_producers=1): # insertion of the new article try: art_contr.create(**new_article) - logger.info('New article added: {}'.format(new_article['link'])) + logger.info("New article added: {}".format(new_article["link"])) except Exception: - logger.exception('Error when inserting article in database.') + logger.exception("Error when inserting article in database.") continue @@ -160,19 +157,20 @@ async def retrieve_feed(queue, users, feed_id=None): Launch the processus. """ for user in users: - logger.info('Starting to retrieve feeds for {}'.format(user.nickname)) + logger.info("Starting to retrieve feeds for {}".format(user.nickname)) filters = {} - filters['user_id'] = user.id + filters["user_id"] = user.id if feed_id is not None: - filters['id'] = feed_id - filters['enabled'] = True - filters['error_count__lt'] = conf.DEFAULT_MAX_ERROR - filters['last_retrieved__lt'] = datetime.now() - \ - timedelta(minutes=conf.FEED_REFRESH_INTERVAL) + filters["id"] = feed_id + filters["enabled"] = True + filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR + filters["last_retrieved__lt"] = datetime.now() - timedelta( + minutes=conf.FEED_REFRESH_INTERVAL + ) feeds = FeedController().read(**filters).all() if feeds == []: - logger.info('No feed to retrieve for {}'.format(user.nickname)) + logger.info("No feed to retrieve for {}".format(user.nickname)) for feed in feeds: articles = await parse_feed(user, feed) diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py index 9891e29f..c2494c79 100644 --- a/newspipe/lib/article_utils.py +++ b/newspipe/lib/article_utils.py @@ -13,69 +13,77 @@ import conf from lib.utils import jarr_get logger = logging.getLogger(__name__) -PROCESSED_DATE_KEYS = {'published', 'created', 'updated'} +PROCESSED_DATE_KEYS = {"published", "created", "updated"} def extract_id(entry): """ extract a value from an entry that will identify it among the other of that feed""" - return entry.get('entry_id') or entry.get('id') or entry['link'] + return entry.get("entry_id") or entry.get("id") or entry["link"] async def construct_article(entry, feed, fields=None, fetch=True): "Safe method to transform a feedparser entry into an article" now = datetime.utcnow() article = {} + def push_in_article(key, value): if not fields or key in fields: article[key] = value - push_in_article('feed_id', feed.id) - push_in_article('user_id', feed.user_id) - push_in_article('entry_id', extract_id(entry)) - push_in_article('retrieved_date', now) - if not fields or 'date' in fields: + + push_in_article("feed_id", feed.id) + push_in_article("user_id", feed.user_id) + push_in_article("entry_id", extract_id(entry)) + push_in_article("retrieved_date", now) + if not fields or "date" in fields: for date_key in PROCESSED_DATE_KEYS: if entry.get(date_key): try: - article['date'] = dateutil.parser.parse(entry[date_key])\ - .astimezone(timezone.utc) + article["date"] = dateutil.parser.parse(entry[date_key]).astimezone( + timezone.utc + ) except Exception as e: logger.exception(e) else: break - push_in_article('content', get_article_content(entry)) - if fields is None or {'link', 'title'}.intersection(fields): + push_in_article("content", get_article_content(entry)) + if fields is None or {"link", "title"}.intersection(fields): link, title = await get_article_details(entry, fetch) - push_in_article('link', link) - push_in_article('title', title) - if 'content' in article: - #push_in_article('content', clean_urls(article['content'], link)) - push_in_article('content', article['content']) - push_in_article('tags', {tag.get('term').strip() - for tag in entry.get('tags', []) \ - if tag and tag.get('term', False)}) + push_in_article("link", link) + push_in_article("title", title) + if "content" in article: + # push_in_article('content', clean_urls(article['content'], link)) + push_in_article("content", article["content"]) + push_in_article( + "tags", + { + tag.get("term").strip() + for tag in entry.get("tags", []) + if tag and tag.get("term", False) + }, + ) return article def get_article_content(entry): - content = '' - if entry.get('content'): - content = entry['content'][0]['value'] - elif entry.get('summary'): - content = entry['summary'] + content = "" + if entry.get("content"): + content = entry["content"][0]["value"] + elif entry.get("summary"): + content = entry["summary"] return content async def get_article_details(entry, fetch=True): - article_link = entry.get('link') - article_title = html.unescape(entry.get('title', '')) + article_link = entry.get("link") + article_title = html.unescape(entry.get("title", "")) if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) response = await jarr_get(article_link, timeout=5) except MissingSchema: split, failed = urlsplit(article_link), False - for scheme in 'https', 'http': + for scheme in "https", "http": new_link = urlunsplit(SplitResult(scheme, *split[1:])) try: response = await jarr_get(new_link, timeout=5) @@ -86,39 +94,44 @@ async def get_article_details(entry, fetch=True): article_link = new_link break if failed: - return article_link, article_title or 'No title' + return article_link, article_title or "No title" except Exception as error: - logger.info("Unable to get the real URL of %s. Won't fix " - "link or title. Error: %s", article_link, error) - return article_link, article_title or 'No title' + logger.info( + "Unable to get the real URL of %s. Won't fix " + "link or title. Error: %s", + article_link, + error, + ) + return article_link, article_title or "No title" article_link = response.url if not article_title: - bs_parsed = BeautifulSoup(response.content, 'html.parser', - parse_only=SoupStrainer('head')) + bs_parsed = BeautifulSoup( + response.content, "html.parser", parse_only=SoupStrainer("head") + ) try: - article_title = bs_parsed.find_all('title')[0].text + article_title = bs_parsed.find_all("title")[0].text except IndexError: # no title pass - return article_link, article_title or 'No title' + return article_link, article_title or "No title" class FiltersAction(Enum): - READ = 'mark as read' - LIKED = 'mark as favorite' - SKIP = 'skipped' + READ = "mark as read" + LIKED = "mark as favorite" + SKIP = "skipped" class FiltersType(Enum): - REGEX = 'regex' - MATCH = 'simple match' - EXACT_MATCH = 'exact match' - TAG_MATCH = 'tag match' - TAG_CONTAINS = 'tag contains' + REGEX = "regex" + MATCH = "simple match" + EXACT_MATCH = "exact match" + TAG_MATCH = "tag match" + TAG_CONTAINS = "tag contains" class FiltersTrigger(Enum): - MATCH = 'match' - NO_MATCH = 'no match' + MATCH = "match" + NO_MATCH = "no match" def process_filters(filters, article, only_actions=None): @@ -129,25 +142,30 @@ def process_filters(filters, article, only_actions=None): for filter_ in filters: match = False try: - pattern = filter_.get('pattern', '') - filter_type = FiltersType(filter_.get('type')) - filter_action = FiltersAction(filter_.get('action')) - filter_trigger = FiltersTrigger(filter_.get('action on')) + pattern = filter_.get("pattern", "") + filter_type = FiltersType(filter_.get("type")) + filter_action = FiltersAction(filter_.get("action")) + filter_trigger = FiltersTrigger(filter_.get("action on")) if filter_type is not FiltersType.REGEX: pattern = pattern.lower() except ValueError: continue if filter_action not in only_actions: - logger.debug('ignoring filter %r' % filter_) + logger.debug("ignoring filter %r" % filter_) continue - if filter_action in {FiltersType.REGEX, FiltersType.MATCH, - FiltersType.EXACT_MATCH} and 'title' not in article: + if ( + filter_action + in {FiltersType.REGEX, FiltersType.MATCH, FiltersType.EXACT_MATCH} + and "title" not in article + ): continue - if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \ - and 'tags' not in article: + if ( + filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} + and "tags" not in article + ): continue - title = article.get('title', '').lower() - tags = [tag.lower() for tag in article.get('tags', [])] + title = article.get("title", "").lower() + tags = [tag.lower() for tag in article.get("tags", [])] if filter_type is FiltersType.REGEX: match = re.match(pattern, title) elif filter_type is FiltersType.MATCH: @@ -158,8 +176,12 @@ def process_filters(filters, article, only_actions=None): match = pattern in tags elif filter_type is FiltersType.TAG_CONTAINS: match = any(pattern in tag for tag in tags) - take_action = match and filter_trigger is FiltersTrigger.MATCH \ - or not match and filter_trigger is FiltersTrigger.NO_MATCH + take_action = ( + match + and filter_trigger is FiltersTrigger.MATCH + or not match + and filter_trigger is FiltersTrigger.NO_MATCH + ) if not take_action: continue @@ -172,15 +194,21 @@ def process_filters(filters, article, only_actions=None): skipped = True if skipped or read or liked: - logger.info("%r applied on %r", filter_action.value, - article.get('link') or article.get('title')) + logger.info( + "%r applied on %r", + filter_action.value, + article.get("link") or article.get("title"), + ) return skipped, read, liked def get_skip_and_ids(entry, feed): - entry_ids = construct_article(entry, feed, - {'entry_id', 'feed_id', 'user_id'}, fetch=False) - skipped, _, _ = process_filters(feed.filters, - construct_article(entry, feed, {'title', 'tags'}, fetch=False), - {FiltersAction.SKIP}) + entry_ids = construct_article( + entry, feed, {"entry_id", "feed_id", "user_id"}, fetch=False + ) + skipped, _, _ = process_filters( + feed.filters, + construct_article(entry, feed, {"title", "tags"}, fetch=False), + {FiltersAction.SKIP}, + ) return skipped, entry_ids diff --git a/newspipe/lib/data.py b/newspipe/lib/data.py index 067a0a04..13843746 100644 --- a/newspipe/lib/data.py +++ b/newspipe/lib/data.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Newspipe - A Web based news aggregator. # Copyright (C) 2010-2018 Cédric Bonhomme - https://www.cedricbonhomme.org @@ -72,18 +72,28 @@ def import_opml(nickname, opml_content): link = subscription.xmlUrl except: continue - if None != Feed.query.filter(Feed.user_id == user.id, Feed.link == link).first(): + if ( + None + != Feed.query.filter( + Feed.user_id == user.id, Feed.link == link + ).first() + ): continue try: site_link = subscription.htmlUrl except: site_link = "" - new_feed = Feed(title=title, description=description, - link=link, site_link=site_link, - enabled=True) + new_feed = Feed( + title=title, + description=description, + link=link, + site_link=site_link, + enabled=True, + ) user.feeds.append(new_feed) nb += 1 return nb + nb = read(subscriptions) db.session.commit() return nb @@ -98,40 +108,53 @@ def import_json(nickname, json_content): nb_feeds, nb_articles = 0, 0 # Create feeds: for feed in json_account: - if None != Feed.query.filter(Feed.user_id == user.id, - Feed.link == feed["link"]).first(): + if ( + None + != Feed.query.filter( + Feed.user_id == user.id, Feed.link == feed["link"] + ).first() + ): continue - new_feed = Feed(title=feed["title"], - description="", - link=feed["link"], - site_link=feed["site_link"], - created_date=datetime.datetime. - fromtimestamp(int(feed["created_date"])), - enabled=feed["enabled"]) + new_feed = Feed( + title=feed["title"], + description="", + link=feed["link"], + site_link=feed["site_link"], + created_date=datetime.datetime.fromtimestamp(int(feed["created_date"])), + enabled=feed["enabled"], + ) user.feeds.append(new_feed) nb_feeds += 1 db.session.commit() # Create articles: for feed in json_account: - user_feed = Feed.query.filter(Feed.user_id == user.id, - Feed.link == feed["link"]).first() + user_feed = Feed.query.filter( + Feed.user_id == user.id, Feed.link == feed["link"] + ).first() if None != user_feed: for article in feed["articles"]: - if None == Article.query.filter(Article.user_id == user.id, - Article.feed_id == user_feed.id, - Article.link == article["link"]).first(): - new_article = Article(entry_id=article["link"], - link=article["link"], - title=article["title"], - content=article["content"], - readed=article["readed"], - like=article["like"], - retrieved_date=datetime.datetime. - fromtimestamp(int(article["retrieved_date"])), - date=datetime.datetime. - fromtimestamp(int(article["date"])), - user_id=user.id, - feed_id=user_feed.id) + if ( + None + == Article.query.filter( + Article.user_id == user.id, + Article.feed_id == user_feed.id, + Article.link == article["link"], + ).first() + ): + new_article = Article( + entry_id=article["link"], + link=article["link"], + title=article["title"], + content=article["content"], + readed=article["readed"], + like=article["like"], + retrieved_date=datetime.datetime.fromtimestamp( + int(article["retrieved_date"]) + ), + date=datetime.datetime.fromtimestamp(int(article["date"])), + user_id=user.id, + feed_id=user_feed.id, + ) user_feed.articles.append(new_article) nb_articles += 1 db.session.commit() @@ -144,23 +167,28 @@ def export_json(user): """ articles = [] for feed in user.feeds: - articles.append({ - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "enabled": feed.enabled, - "created_date": feed.created_date.strftime('%s'), - "articles": [ { - "title": article.title, - "link": article.link, - "content": article.content, - "readed": article.readed, - "like": article.like, - "date": article.date.strftime('%s'), - "retrieved_date": article.retrieved_date.strftime('%s') - } for article in feed.articles] - }) + articles.append( + { + "title": feed.title, + "description": feed.description, + "link": feed.link, + "site_link": feed.site_link, + "enabled": feed.enabled, + "created_date": feed.created_date.strftime("%s"), + "articles": [ + { + "title": article.title, + "link": article.link, + "content": article.content, + "readed": article.readed, + "like": article.like, + "date": article.date.strftime("%s"), + "retrieved_date": article.retrieved_date.strftime("%s"), + } + for article in feed.articles + ], + } + ) return jsonify(articles) @@ -173,19 +201,18 @@ def import_pinboard_json(user, json_content): nb_bookmarks = 0 for bookmark in bookmarks: tags = [] - for tag in bookmark['tags'].split(' '): + for tag in bookmark["tags"].split(" "): new_tag = BookmarkTag(text=tag.strip(), user_id=user.id) tags.append(new_tag) bookmark_attr = { - 'href': bookmark['href'], - 'description': bookmark['extended'], - 'title': bookmark['description'], - 'shared': [bookmark['shared']=='yes' and True or False][0], - 'to_read': [bookmark['toread']=='yes' and True or False][0], - 'time': datetime.datetime.strptime(bookmark['time'], - '%Y-%m-%dT%H:%M:%SZ'), - 'tags': tags - } + "href": bookmark["href"], + "description": bookmark["extended"], + "title": bookmark["description"], + "shared": [bookmark["shared"] == "yes" and True or False][0], + "to_read": [bookmark["toread"] == "yes" and True or False][0], + "time": datetime.datetime.strptime(bookmark["time"], "%Y-%m-%dT%H:%M:%SZ"), + "tags": tags, + } new_bookmark = bookmark_contr.create(**bookmark_attr) nb_bookmarks += 1 return nb_bookmarks @@ -198,13 +225,15 @@ def export_bookmarks(user): bookmarks = bookmark_contr.read() export = [] for bookmark in bookmarks: - export.append({ - 'href': bookmark.href, - 'description': bookmark.description, - 'title': bookmark.title, - 'shared': 'yes' if bookmark.shared else 'no', - 'toread': 'yes' if bookmark.to_read else 'no', - 'time': bookmark.time.isoformat(), - 'tags': ' '.join(bookmark.tags_proxy) - }) + export.append( + { + "href": bookmark.href, + "description": bookmark.description, + "title": bookmark.title, + "shared": "yes" if bookmark.shared else "no", + "toread": "yes" if bookmark.to_read else "no", + "time": bookmark.time.isoformat(), + "tags": " ".join(bookmark.tags_proxy), + } + ) return jsonify(export) diff --git a/newspipe/lib/feed_utils.py b/newspipe/lib/feed_utils.py index c2d4ca6e..9f1e2354 100644 --- a/newspipe/lib/feed_utils.py +++ b/newspipe/lib/feed_utils.py @@ -10,12 +10,17 @@ from lib.utils import try_keys, try_get_icon_url, rebuild_url logger = logging.getLogger(__name__) logging.captureWarnings(True) -ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml', - 'application/atom+xml', 'application/xml', 'text/xml') +ACCEPTED_MIMETYPES = ( + "application/rss+xml", + "application/rdf+xml", + "application/atom+xml", + "application/xml", + "text/xml", +) def is_parsing_ok(parsed_feed): - return parsed_feed['entries'] or not parsed_feed['bozo'] + return parsed_feed["entries"] or not parsed_feed["bozo"] def escape_keys(*keys): @@ -24,66 +29,71 @@ def escape_keys(*keys): result = func(*args, **kwargs) for key in keys: if key in result: - result[key] = html.unescape(result[key] or '') + result[key] = html.unescape(result[key] or "") return result + return metawrapper + return wrapper -@escape_keys('title', 'description') +@escape_keys("title", "description") def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): - requests_kwargs = {'headers': {'User-Agent': CRAWLER_USER_AGENT}, - 'verify': False} + requests_kwargs = {"headers": {"User-Agent": CRAWLER_USER_AGENT}, "verify": False} if url is None and fp_parsed is not None: - url = fp_parsed.get('url') + url = fp_parsed.get("url") if url is not None and fp_parsed is None: try: response = requests.get(url, **requests_kwargs) - fp_parsed = feedparser.parse(response.content, - request_headers=response.headers) + fp_parsed = feedparser.parse( + response.content, request_headers=response.headers + ) except Exception: - logger.exception('failed to retrieve that url') - fp_parsed = {'bozo': True} + logger.exception("failed to retrieve that url") + fp_parsed = {"bozo": True} assert url is not None and fp_parsed is not None feed = feed or {} feed_split = urllib.parse.urlsplit(url) site_split = None if is_parsing_ok(fp_parsed): - feed['link'] = url - feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link') - feed['title'] = fp_parsed['feed'].get('title') - feed['description'] = try_keys(fp_parsed['feed'], 'subtitle', 'title') - feed['icon_url'] = try_keys(fp_parsed['feed'], 'icon') + feed["link"] = url + feed["site_link"] = try_keys(fp_parsed["feed"], "href", "link") + feed["title"] = fp_parsed["feed"].get("title") + feed["description"] = try_keys(fp_parsed["feed"], "subtitle", "title") + feed["icon_url"] = try_keys(fp_parsed["feed"], "icon") else: - feed['site_link'] = url - - if feed.get('site_link'): - feed['site_link'] = rebuild_url(feed['site_link'], feed_split) - site_split = urllib.parse.urlsplit(feed['site_link']) - - if feed.get('icon_url'): - feed['icon_url'] = try_get_icon_url( - feed['icon_url'], site_split, feed_split) - if feed['icon_url'] is None: - del feed['icon_url'] - - if not feed.get('site_link') or not query_site \ - or all(bool(feed.get(k)) for k in ('link', 'title', 'icon_url')): + feed["site_link"] = url + + if feed.get("site_link"): + feed["site_link"] = rebuild_url(feed["site_link"], feed_split) + site_split = urllib.parse.urlsplit(feed["site_link"]) + + if feed.get("icon_url"): + feed["icon_url"] = try_get_icon_url(feed["icon_url"], site_split, feed_split) + if feed["icon_url"] is None: + del feed["icon_url"] + + if ( + not feed.get("site_link") + or not query_site + or all(bool(feed.get(k)) for k in ("link", "title", "icon_url")) + ): return feed try: - response = requests.get(feed['site_link'], **requests_kwargs) + response = requests.get(feed["site_link"], **requests_kwargs) except requests.exceptions.InvalidSchema as e: return feed except: - logger.exception('failed to retrieve %r', feed['site_link']) + logger.exception("failed to retrieve %r", feed["site_link"]) return feed - bs_parsed = BeautifulSoup(response.content, 'html.parser', - parse_only=SoupStrainer('head')) + bs_parsed = BeautifulSoup( + response.content, "html.parser", parse_only=SoupStrainer("head") + ) - if not feed.get('title'): + if not feed.get("title"): try: - feed['title'] = bs_parsed.find_all('title')[0].text + feed["title"] = bs_parsed.find_all("title")[0].text except Exception: pass @@ -95,31 +105,30 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): if not all(val in elem.attrs[key] for val in vals): return False return True + return wrapper - if not feed.get('icon_url'): - icons = bs_parsed.find_all(check_keys(rel=['icon', 'shortcut'])) + if not feed.get("icon_url"): + icons = bs_parsed.find_all(check_keys(rel=["icon", "shortcut"])) if not len(icons): - icons = bs_parsed.find_all(check_keys(rel=['icon'])) + icons = bs_parsed.find_all(check_keys(rel=["icon"])) if len(icons) >= 1: for icon in icons: - feed['icon_url'] = try_get_icon_url(icon.attrs['href'], - site_split, feed_split) - if feed['icon_url'] is not None: + feed["icon_url"] = try_get_icon_url( + icon.attrs["href"], site_split, feed_split + ) + if feed["icon_url"] is not None: break - if feed.get('icon_url') is None: - feed['icon_url'] = try_get_icon_url('/favicon.ico', - site_split, feed_split) - if 'icon_url' in feed and feed['icon_url'] is None: - del feed['icon_url'] + if feed.get("icon_url") is None: + feed["icon_url"] = try_get_icon_url("/favicon.ico", site_split, feed_split) + if "icon_url" in feed and feed["icon_url"] is None: + del feed["icon_url"] - if not feed.get('link'): + if not feed.get("link"): for type_ in ACCEPTED_MIMETYPES: - alternates = bs_parsed.find_all(check_keys( - rel=['alternate'], type=[type_])) + alternates = bs_parsed.find_all(check_keys(rel=["alternate"], type=[type_])) if len(alternates) >= 1: - feed['link'] = rebuild_url(alternates[0].attrs['href'], - feed_split) + feed["link"] = rebuild_url(alternates[0].attrs["href"], feed_split) break return feed diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py index 8fb2d284..6fd590ac 100755 --- a/newspipe/lib/misc_utils.py +++ b/newspipe/lib/misc_utils.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Newspipe - A Web based news aggregator. # Copyright (C) 2010-2018 Cédric Bonhomme - https://www.cedricbonhomme.org @@ -36,6 +36,7 @@ import operator import urllib import subprocess import sqlalchemy + try: from urlparse import urlparse, parse_qs, urlunparse except: @@ -50,7 +51,7 @@ from lib.utils import clear_string logger = logging.getLogger(__name__) -ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) +ALLOWED_EXTENSIONS = set(["xml", "opml", "json"]) def is_safe_url(target): @@ -59,15 +60,14 @@ def is_safe_url(target): """ ref_url = urlparse(request.host_url) test_url = urlparse(urljoin(request.host_url, target)) - return test_url.scheme in ('http', 'https') and \ - ref_url.netloc == test_url.netloc + return test_url.scheme in ("http", "https") and ref_url.netloc == test_url.netloc def get_redirect_target(): """ Looks at various hints to find the redirect target. """ - for target in request.args.get('next'), request.referrer: + for target in request.args.get("next"), request.referrer: if not target: continue if is_safe_url(target): @@ -78,8 +78,7 @@ def allowed_file(filename): """ Check if the uploaded file is allowed. """ - return '.' in filename and \ - filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS + return "." in filename and filename.rsplit(".", 1)[1] in ALLOWED_EXTENSIONS @contextmanager @@ -100,10 +99,14 @@ def fetch(id, feed_id=None): Fetch the feeds in a new processus. The default crawler ("asyncio") is launched with the manager. """ - cmd = [sys.executable, conf.BASE_DIR + '/manager.py', 'fetch_asyncio', - '--user_id='+str(id)] + cmd = [ + sys.executable, + conf.BASE_DIR + "/manager.py", + "fetch_asyncio", + "--user_id=" + str(id), + ] if feed_id: - cmd.append('--feed_id='+str(feed_id)) + cmd.append("--feed_id=" + str(feed_id)) return subprocess.Popen(cmd, stdout=subprocess.PIPE) @@ -114,9 +117,11 @@ def history(user_id, year=None, month=None): articles_counter = Counter() articles = ArticleController(user_id).read() if None != year: - articles = articles.filter(sqlalchemy.extract('year', 'Article.date') == year) + articles = articles.filter(sqlalchemy.extract("year", "Article.date") == year) if None != month: - articles = articles.filter(sqlalchemy.extract('month', 'Article.date') == month) + articles = articles.filter( + sqlalchemy.extract("month", "Article.date") == month + ) for article in articles.all(): if None != year: articles_counter[article.date.month] += 1 @@ -131,24 +136,26 @@ def clean_url(url): """ parsed_url = urlparse(url) qd = parse_qs(parsed_url.query, keep_blank_values=True) - filtered = dict((k, v) for k, v in qd.items() - if not k.startswith('utm_')) - return urlunparse([ - parsed_url.scheme, - parsed_url.netloc, - urllib.parse.quote(urllib.parse.unquote(parsed_url.path)), - parsed_url.params, - urllib.parse.urlencode(filtered, doseq=True), - parsed_url.fragment - ]).rstrip('=') + filtered = dict((k, v) for k, v in qd.items() if not k.startswith("utm_")) + return urlunparse( + [ + parsed_url.scheme, + parsed_url.netloc, + urllib.parse.quote(urllib.parse.unquote(parsed_url.path)), + parsed_url.params, + urllib.parse.urlencode(filtered, doseq=True), + parsed_url.fragment, + ] + ).rstrip("=") def load_stop_words(): """ Load the stop words and return them in a list. """ - stop_words_lists = glob.glob(os.path.join(conf.BASE_DIR, - 'web/var/stop_words/*.txt')) + stop_words_lists = glob.glob( + os.path.join(conf.BASE_DIR, "web/var/stop_words/*.txt") + ) stop_words = [] for stop_wods_list in stop_words_lists: @@ -166,11 +173,13 @@ def top_words(articles, n=10, size=5): """ stop_words = load_stop_words() words = Counter() - wordre = re.compile(r'\b\w{%s,}\b' % size, re.I) + wordre = re.compile(r"\b\w{%s,}\b" % size, re.I) for article in articles: - for word in [elem.lower() for elem in - wordre.findall(clear_string(article.content)) \ - if elem.lower() not in stop_words]: + for word in [ + elem.lower() + for elem in wordre.findall(clear_string(article.content)) + if elem.lower() not in stop_words + ]: words[word] += 1 return words.most_common(n) @@ -181,5 +190,9 @@ def tag_cloud(tags): """ tags.sort(key=operator.itemgetter(0)) max_tag = max([tag[1] for tag in tags]) - return '\n'.join([('<font size=%d>%s</font>' % \ - (min(1 + count * 7 / max_tag, 7), word)) for (word, count) in tags]) + return "\n".join( + [ + ("<font size=%d>%s</font>" % (min(1 + count * 7 / max_tag, 7), word)) + for (word, count) in tags + ] + ) diff --git a/newspipe/lib/utils.py b/newspipe/lib/utils.py index d206b769..f7244e17 100644 --- a/newspipe/lib/utils.py +++ b/newspipe/lib/utils.py @@ -11,18 +11,20 @@ import conf logger = logging.getLogger(__name__) -def default_handler(obj, role='admin'): +def default_handler(obj, role="admin"): """JSON handler for default query formatting""" - if hasattr(obj, 'isoformat'): + if hasattr(obj, "isoformat"): return obj.isoformat() - if hasattr(obj, 'dump'): + if hasattr(obj, "dump"): return obj.dump(role=role) if isinstance(obj, (set, frozenset, types.GeneratorType)): return list(obj) if isinstance(obj, BaseException): return str(obj) - raise TypeError("Object of type %s with value of %r " - "is not JSON serializable" % (type(obj), obj)) + raise TypeError( + "Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj) + ) def try_keys(dico, *keys): @@ -37,9 +39,12 @@ def rebuild_url(url, base_split): if split.scheme and split.netloc: return url # url is fine new_split = urllib.parse.SplitResult( - scheme=split.scheme or base_split.scheme, - netloc=split.netloc or base_split.netloc, - path=split.path, query='', fragment='') + scheme=split.scheme or base_split.scheme, + netloc=split.netloc or base_split.netloc, + path=split.path, + query="", + fragment="", + ) return urllib.parse.urlunsplit(new_split) @@ -52,19 +57,22 @@ def try_get_icon_url(url, *splits): # if html in content-type, we assume it's a fancy 404 page try: response = jarr_get(rb_url) - content_type = response.headers.get('content-type', '') + content_type = response.headers.get("content-type", "") except Exception: pass else: - if response is not None and response.ok \ - and 'html' not in content_type and response.content: + if ( + response is not None + and response.ok + and "html" not in content_type + and response.content + ): return response.url return None def to_hash(text): - return md5(text.encode('utf8') if hasattr(text, 'encode') else text)\ - .hexdigest() + return md5(text.encode("utf8") if hasattr(text, "encode") else text).hexdigest() def clear_string(data): @@ -72,18 +80,21 @@ def clear_string(data): Clear a string by removing HTML tags, HTML special caracters and consecutive white spaces (more that one). """ - p = re.compile('<[^>]+>') # HTML tags - q = re.compile('\s') # consecutive white spaces - return p.sub('', q.sub(' ', data)) + p = re.compile("<[^>]+>") # HTML tags + q = re.compile("\s") # consecutive white spaces + return p.sub("", q.sub(" ", data)) -def redirect_url(default='home'): - return request.args.get('next') or request.referrer or url_for(default) +def redirect_url(default="home"): + return request.args.get("next") or request.referrer or url_for(default) async def jarr_get(url, **kwargs): - request_kwargs = {'verify': False, 'allow_redirects': True, - 'timeout': conf.CRAWLER_TIMEOUT, - 'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}} + request_kwargs = { + "verify": False, + "allow_redirects": True, + "timeout": conf.CRAWLER_TIMEOUT, + "headers": {"User-Agent": conf.CRAWLER_USER_AGENT}, + } request_kwargs.update(kwargs) return requests.get(url, **request_kwargs) diff --git a/newspipe/manager.py b/newspipe/manager.py index 9535ac59..60f4c729 100755 --- a/newspipe/manager.py +++ b/newspipe/manager.py @@ -12,12 +12,12 @@ from flask_migrate import Migrate, MigrateCommand import web.models from web.controllers import UserController -logger = logging.getLogger('manager') +logger = logging.getLogger("manager") Migrate(application, db) manager = Manager(application) -manager.add_command('db', MigrateCommand) +manager.add_command("db", MigrateCommand) @manager.command @@ -30,23 +30,32 @@ def db_empty(): @manager.command def db_create(): "Will create the database from conf parameters." - admin = {'is_admin': True, 'is_api': True, 'is_active': True, - 'nickname': 'admin', - 'pwdhash': generate_password_hash( - os.environ.get("ADMIN_PASSWORD", "password"))} + admin = { + "is_admin": True, + "is_api": True, + "is_active": True, + "nickname": "admin", + "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")), + } with application.app_context(): db.create_all() UserController(ignore_context=True).create(**admin) + @manager.command def create_admin(nickname, password): "Will create an admin user." - admin = {'is_admin': True, 'is_api': True, 'is_active': True, - 'nickname': nickname, - 'pwdhash': generate_password_hash(password)} + admin = { + "is_admin": True, + "is_api": True, + "is_active": True, + "nickname": nickname, + "pwdhash": generate_password_hash(password), + } with application.app_context(): UserController(ignore_context=True).create(**admin) + @manager.command def fetch_asyncio(user_id=None, feed_id=None): "Crawl the feeds with asyncio." @@ -54,33 +63,32 @@ def fetch_asyncio(user_id=None, feed_id=None): with application.app_context(): from crawler import default_crawler + filters = {} - filters['is_active'] = True - filters['automatic_crawling'] = True + filters["is_active"] = True + filters["automatic_crawling"] = True if None is not user_id: - filters['id'] = user_id + filters["id"] = user_id users = UserController().read(**filters).all() try: feed_id = int(feed_id) except: feed_id = None - - + loop = asyncio.get_event_loop() queue = asyncio.Queue(maxsize=3, loop=loop) - + producer_coro = default_crawler.retrieve_feed(queue, users, feed_id) consumer_coro = default_crawler.insert_articles(queue, 1) - logger.info('Starting crawler.') + logger.info("Starting crawler.") start = datetime.now() loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro)) end = datetime.now() loop.close() - logger.info('Crawler finished in {} seconds.' \ - .format((end - start).seconds)) + logger.info("Crawler finished in {} seconds.".format((end - start).seconds)) -if __name__ == '__main__': +if __name__ == "__main__": manager.run() diff --git a/newspipe/notifications/emails.py b/newspipe/notifications/emails.py index 90c87c93..1d156bd8 100644 --- a/newspipe/notifications/emails.py +++ b/newspipe/notifications/emails.py @@ -36,31 +36,33 @@ def send_async_email(mfrom, mto, msg): s = smtplib.SMTP(conf.NOTIFICATION_HOST) s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) except Exception: - logger.exception('send_async_email raised:') + logger.exception("send_async_email raised:") else: s.sendmail(mfrom, mto, msg.as_string()) s.quit() + def send(*args, **kwargs): """ This functions enables to send email via different method. """ send_smtp(**kwargs) + def send_smtp(to="", bcc="", subject="", plaintext="", html=""): """ Send an email. """ # Create message container - the correct MIME type is multipart/alternative. - msg = MIMEMultipart('alternative') - msg['Subject'] = subject - msg['From'] = conf.NOTIFICATION_EMAIL - msg['To'] = to - msg['BCC'] = bcc + msg = MIMEMultipart("alternative") + msg["Subject"] = subject + msg["From"] = conf.NOTIFICATION_EMAIL + msg["To"] = to + msg["BCC"] = bcc # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(plaintext, 'plain', 'utf-8') - part2 = MIMEText(html, 'html', 'utf-8') + part1 = MIMEText(plaintext, "plain", "utf-8") + part2 = MIMEText(html, "html", "utf-8") # Attach parts into message container. # According to RFC 2046, the last part of a multipart message, in this case @@ -74,5 +76,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""): except Exception: logger.exception("send_smtp raised:") else: - s.sendmail(conf.NOTIFICATION_EMAIL, msg['To'] + ", " + msg['BCC'], msg.as_string()) + s.sendmail( + conf.NOTIFICATION_EMAIL, msg["To"] + ", " + msg["BCC"], msg.as_string() + ) s.quit() diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py index e775f4b9..2bc24810 100644 --- a/newspipe/notifications/notifications.py +++ b/newspipe/notifications/notifications.py @@ -31,23 +31,34 @@ def new_account_notification(user, email): Account creation notification. """ token = generate_confirmation_token(user.nickname) - expire_time = datetime.datetime.now() + \ - datetime.timedelta(seconds=conf.TOKEN_VALIDITY_PERIOD) + expire_time = datetime.datetime.now() + datetime.timedelta( + seconds=conf.TOKEN_VALIDITY_PERIOD + ) - plaintext = render_template('emails/account_activation.txt', - user=user, platform_url=conf.PLATFORM_URL, - token=token, - expire_time=expire_time) + plaintext = render_template( + "emails/account_activation.txt", + user=user, + platform_url=conf.PLATFORM_URL, + token=token, + expire_time=expire_time, + ) + + emails.send( + to=email, + bcc=conf.NOTIFICATION_EMAIL, + subject="[Newspipe] Account creation", + plaintext=plaintext, + ) - emails.send(to=email, bcc=conf.NOTIFICATION_EMAIL, - subject="[Newspipe] Account creation", plaintext=plaintext) def new_password_notification(user, password): """ New password notification. """ - plaintext = render_template('emails/new_password.txt', - user=user, password=password) - emails.send(to=user.email, - bcc=conf.NOTIFICATION_EMAIL, - subject="[Newspipe] New password", plaintext=plaintext) + plaintext = render_template("emails/new_password.txt", user=user, password=password) + emails.send( + to=user.email, + bcc=conf.NOTIFICATION_EMAIL, + subject="[Newspipe] New password", + plaintext=plaintext, + ) diff --git a/newspipe/runserver.py b/newspipe/runserver.py index a1ebb54c..2b016f1c 100755 --- a/newspipe/runserver.py +++ b/newspipe/runserver.py @@ -29,9 +29,11 @@ babel = Babel(application) # Jinja filters def month_name(month_number): return calendar.month_name[month_number] -application.jinja_env.filters['month_name'] = month_name -application.jinja_env.filters['datetime'] = format_datetime -application.jinja_env.globals['conf'] = conf + + +application.jinja_env.filters["month_name"] = month_name +application.jinja_env.filters["datetime"] = format_datetime +application.jinja_env.globals["conf"] = conf # Views from flask_restful import Api @@ -39,10 +41,11 @@ from flask import g with application.app_context(): populate_g() - g.api = Api(application, prefix='/api/v2.0') + g.api = Api(application, prefix="/api/v2.0") g.babel = babel from web import views + application.register_blueprint(views.articles_bp) application.register_blueprint(views.article_bp) application.register_blueprint(views.feeds_bp) @@ -57,7 +60,7 @@ with application.app_context(): application.register_blueprint(views.bookmark_bp) -if __name__ == '__main__': - application.run(host=conf.WEBSERVER_HOST, - port=conf.WEBSERVER_PORT, - debug=conf.WEBSERVER_DEBUG) +if __name__ == "__main__": + application.run( + host=conf.WEBSERVER_HOST, port=conf.WEBSERVER_PORT, debug=conf.WEBSERVER_DEBUG + ) diff --git a/newspipe/web/controllers/__init__.py b/newspipe/web/controllers/__init__.py index 5fbc2619..9b193cc5 100644 --- a/newspipe/web/controllers/__init__.py +++ b/newspipe/web/controllers/__init__.py @@ -7,6 +7,12 @@ from .bookmark import BookmarkController from .tag import BookmarkTagController -__all__ = ['FeedController', 'CategoryController', 'ArticleController', - 'UserController', 'IconController', 'BookmarkController', - 'BookmarkTagController'] +__all__ = [ + "FeedController", + "CategoryController", + "ArticleController", + "UserController", + "IconController", + "BookmarkController", + "BookmarkTagController", +] diff --git a/newspipe/web/controllers/abstract.py b/newspipe/web/controllers/abstract.py index 764ff305..9d9e84f2 100644 --- a/newspipe/web/controllers/abstract.py +++ b/newspipe/web/controllers/abstract.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) class AbstractController: _db_cls = None # reference to the database class - _user_id_key = 'user_id' + _user_id_key = "user_id" def __init__(self, user_id=None, ignore_context=False): """User id is a right management mechanism that should be used to @@ -36,25 +36,25 @@ class AbstractController: """ db_filters = set() for key, value in filters.items(): - if key == '__or__': + if key == "__or__": db_filters.add(or_(*self._to_filters(**value))) - elif key.endswith('__gt'): + elif key.endswith("__gt"): db_filters.add(getattr(self._db_cls, key[:-4]) > value) - elif key.endswith('__lt'): + elif key.endswith("__lt"): db_filters.add(getattr(self._db_cls, key[:-4]) < value) - elif key.endswith('__ge'): + elif key.endswith("__ge"): db_filters.add(getattr(self._db_cls, key[:-4]) >= value) - elif key.endswith('__le'): + elif key.endswith("__le"): db_filters.add(getattr(self._db_cls, key[:-4]) <= value) - elif key.endswith('__ne'): + elif key.endswith("__ne"): db_filters.add(getattr(self._db_cls, key[:-4]) != value) - elif key.endswith('__in'): + elif key.endswith("__in"): db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) - elif key.endswith('__contains'): + elif key.endswith("__contains"): db_filters.add(getattr(self._db_cls, key[:-10]).contains(value)) - elif key.endswith('__like'): + elif key.endswith("__like"): db_filters.add(getattr(self._db_cls, key[:-6]).like(value)) - elif key.endswith('__ilike'): + elif key.endswith("__ilike"): db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value)) else: db_filters.add(getattr(self._db_cls, key) == value) @@ -66,8 +66,11 @@ class AbstractController: dependent) and the user is not an admin and the filters doesn't already contains a filter for that user. """ - if self._user_id_key is not None and self.user_id \ - and filters.get(self._user_id_key) != self.user_id: + if ( + self._user_id_key is not None + and self.user_id + and filters.get(self._user_id_key) != self.user_id + ): filters[self._user_id_key] = self.user_id return self._db_cls.query.filter(*self._to_filters(**filters)) @@ -76,20 +79,27 @@ class AbstractController: obj = self._get(**filters).first() if obj and not self._has_right_on(obj): - raise Forbidden({'message': 'No authorized to access %r (%r)' - % (self._db_cls.__class__.__name__, filters)}) + raise Forbidden( + { + "message": "No authorized to access %r (%r)" + % (self._db_cls.__class__.__name__, filters) + } + ) if not obj: - raise NotFound({'message': 'No %r (%r)' - % (self._db_cls.__class__.__name__, filters)}) + raise NotFound( + {"message": "No %r (%r)" % (self._db_cls.__class__.__name__, filters)} + ) return obj def create(self, **attrs): assert attrs, "attributes to update must not be empty" if self._user_id_key is not None and self._user_id_key not in attrs: attrs[self._user_id_key] = self.user_id - assert self._user_id_key is None or self._user_id_key in attrs \ - or self.user_id is None, \ - "You must provide user_id one way or another" + assert ( + self._user_id_key is None + or self._user_id_key in attrs + or self.user_id is None + ), "You must provide user_id one way or another" obj = self._db_cls(**attrs) db.session.add(obj) @@ -123,39 +133,45 @@ class AbstractController: # user_id == None is like being admin if self._user_id_key is None: return True - return self.user_id is None \ - or getattr(obj, self._user_id_key, None) == self.user_id + return ( + self.user_id is None + or getattr(obj, self._user_id_key, None) == self.user_id + ) def _count_by(self, elem_to_group_by, filters): if self.user_id: - filters['user_id'] = self.user_id - return dict(db.session.query(elem_to_group_by, func.count('id')) - .filter(*self._to_filters(**filters)) - .group_by(elem_to_group_by).all()) + filters["user_id"] = self.user_id + return dict( + db.session.query(elem_to_group_by, func.count("id")) + .filter(*self._to_filters(**filters)) + .group_by(elem_to_group_by) + .all() + ) @classmethod def _get_attrs_desc(cls, role, right=None): result = defaultdict(dict) - if role == 'admin': + if role == "admin": columns = cls._db_cls.__table__.columns.keys() else: - assert role in {'base', 'api'}, 'unknown role %r' % role - assert right in {'read', 'write'}, \ - "right must be 'read' or 'write' with role %r" % role - columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() + assert role in {"base", "api"}, "unknown role %r" % role + assert right in {"read", "write"}, ( + "right must be 'read' or 'write' with role %r" % role + ) + columns = getattr(cls._db_cls, "fields_%s_%s" % (role, right))() for column in columns: result[column] = {} db_col = getattr(cls._db_cls, column).property.columns[0] try: - result[column]['type'] = db_col.type.python_type + result[column]["type"] = db_col.type.python_type except NotImplementedError: if db_col.default: - result[column]['type'] = db_col.default.arg.__class__ + result[column]["type"] = db_col.default.arg.__class__ if column not in result: continue - if issubclass(result[column]['type'], datetime): - result[column]['default'] = datetime.utcnow() - result[column]['type'] = lambda x: dateutil.parser.parse(x) + if issubclass(result[column]["type"], datetime): + result[column]["default"] = datetime.utcnow() + result[column]["type"] = lambda x: dateutil.parser.parse(x) elif db_col.default: - result[column]['default'] = db_col.default.arg + result[column]["default"] = db_col.default.arg return result diff --git a/newspipe/web/controllers/article.py b/newspipe/web/controllers/article.py index d7058229..03342a1f 100644 --- a/newspipe/web/controllers/article.py +++ b/newspipe/web/controllers/article.py @@ -30,19 +30,24 @@ class ArticleController(AbstractController): return self._count_by(Article.feed_id, filters) def count_by_user_id(self, **filters): - return dict(db.session.query(Article.user_id, func.count(Article.id)) - .filter(*self._to_filters(**filters)) - .group_by(Article.user_id).all()) + return dict( + db.session.query(Article.user_id, func.count(Article.id)) + .filter(*self._to_filters(**filters)) + .group_by(Article.user_id) + .all() + ) def create(self, **attrs): # handling special denorm for article rights - assert 'feed_id' in attrs, "must provide feed_id when creating article" - feed = FeedController( - attrs.get('user_id', self.user_id)).get(id=attrs['feed_id']) - if 'user_id' in attrs: - assert feed.user_id == attrs['user_id'] or self.user_id is None, \ - "no right on feed %r" % feed.id - attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id + assert "feed_id" in attrs, "must provide feed_id when creating article" + feed = FeedController(attrs.get("user_id", self.user_id)).get( + id=attrs["feed_id"] + ) + if "user_id" in attrs: + assert feed.user_id == attrs["user_id"] or self.user_id is None, ( + "no right on feed %r" % feed.id + ) + attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id skipped, read, liked = process_filters(feed.filters, attrs) if skipped: @@ -51,15 +56,16 @@ class ArticleController(AbstractController): return article def update(self, filters, attrs): - user_id = attrs.get('user_id', self.user_id) - if 'feed_id' in attrs: - feed = FeedController().get(id=attrs['feed_id']) + user_id = attrs.get("user_id", self.user_id) + if "feed_id" in attrs: + feed = FeedController().get(id=attrs["feed_id"]) assert feed.user_id == user_id, "no right on feed %r" % feed.id - attrs['category_id'] = feed.category_id - if attrs.get('category_id'): - cat = CategoryController().get(id=attrs['category_id']) - assert self.user_id is None or cat.user_id == user_id, \ - "no right on cat %r" % cat.id + attrs["category_id"] = feed.category_id + if attrs.get("category_id"): + cat = CategoryController().get(id=attrs["category_id"]) + assert self.user_id is None or cat.user_id == user_id, ( + "no right on cat %r" % cat.id + ) return super().update(filters, attrs) def get_history(self, year=None, month=None): @@ -69,11 +75,11 @@ class ArticleController(AbstractController): articles_counter = Counter() articles = self.read() if year is not None: - articles = articles.filter( - sqlalchemy.extract('year', Article.date) == year) + articles = articles.filter(sqlalchemy.extract("year", Article.date) == year) if month is not None: articles = articles.filter( - sqlalchemy.extract('month', Article.date) == month) + sqlalchemy.extract("month", Article.date) == month + ) for article in articles.all(): if year is not None: articles_counter[article.date.month] += 1 @@ -82,6 +88,17 @@ class ArticleController(AbstractController): return articles_counter, articles def read_light(self, **filters): - return super().read(**filters).with_entities(Article.id, Article.title, - Article.readed, Article.like, Article.feed_id, Article.date, - Article.category_id).order_by(Article.date.desc()) + return ( + super() + .read(**filters) + .with_entities( + Article.id, + Article.title, + Article.readed, + Article.like, + Article.feed_id, + Article.date, + Article.category_id, + ) + .order_by(Article.date.desc()) + ) diff --git a/newspipe/web/controllers/bookmark.py b/newspipe/web/controllers/bookmark.py index b5413243..d1c1260c 100644 --- a/newspipe/web/controllers/bookmark.py +++ b/newspipe/web/controllers/bookmark.py @@ -17,16 +17,19 @@ class BookmarkController(AbstractController): return self._count_by(Bookmark.href, filters) def update(self, filters, attrs): - BookmarkTagController(self.user_id) \ - .read(**{'bookmark_id': filters["id"]}) \ - .delete() + BookmarkTagController(self.user_id).read( + **{"bookmark_id": filters["id"]} + ).delete() - for tag in attrs['tags']: + for tag in attrs["tags"]: BookmarkTagController(self.user_id).create( - **{'text': tag.text, - 'id': tag.id, - 'bookmark_id': tag.bookmark_id, - 'user_id': tag.user_id}) - - del attrs['tags'] + **{ + "text": tag.text, + "id": tag.id, + "bookmark_id": tag.bookmark_id, + "user_id": tag.user_id, + } + ) + + del attrs["tags"] return super().update(filters, attrs) diff --git a/newspipe/web/controllers/category.py b/newspipe/web/controllers/category.py index fef5ca81..ec54f5a3 100644 --- a/newspipe/web/controllers/category.py +++ b/newspipe/web/controllers/category.py @@ -7,6 +7,7 @@ class CategoryController(AbstractController): _db_cls = Category def delete(self, obj_id): - FeedController(self.user_id).update({'category_id': obj_id}, - {'category_id': None}) + FeedController(self.user_id).update( + {"category_id": obj_id}, {"category_id": None} + ) return super().delete(obj_id) diff --git a/newspipe/web/controllers/feed.py b/newspipe/web/controllers/feed.py index d75cd994..19ba463f 100644 --- a/newspipe/web/controllers/feed.py +++ b/newspipe/web/controllers/feed.py @@ -16,22 +16,26 @@ DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR class FeedController(AbstractController): _db_cls = Feed - def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, - limit=DEFAULT_LIMIT): - return [feed for feed in self.read( - error_count__lt=max_error, enabled=True, - last_retrieved__lt=max_last) - .join(User).filter(User.is_active == True) - .order_by('last_retrieved') - .limit(limit)] + def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + return [ + feed + for feed in self.read( + error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last + ) + .join(User) + .filter(User.is_active == True) + .order_by("last_retrieved") + .limit(limit) + ] def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): now = datetime.now() max_last = now - timedelta(minutes=60) feeds = self.list_late(max_last, max_error, limit) if feeds: - self.update({'id__in': [feed.id for feed in feeds]}, - {'last_retrieved': now}) + self.update( + {"id__in": [feed.id for feed in feeds]}, {"last_retrieved": now} + ) return feeds def get_duplicates(self, feed_id): @@ -43,8 +47,9 @@ class FeedController(AbstractController): duplicates = [] for pair in itertools.combinations(feed.articles[:1000], 2): date1, date2 = pair[0].date, pair[1].date - if clear_string(pair[0].title) == clear_string(pair[1].title) \ - and (date1 - date2) < timedelta(days=1): + if clear_string(pair[0].title) == clear_string(pair[1].title) and ( + date1 - date2 + ) < timedelta(days=1): if pair[0].retrieved_date < pair[1].retrieved_date: duplicates.append((pair[0], pair[1])) else: @@ -75,11 +80,11 @@ class FeedController(AbstractController): return self._count_by(Feed.link, filters) def _ensure_icon(self, attrs): - if not attrs.get('icon_url'): + if not attrs.get("icon_url"): return icon_contr = IconController() - if not icon_contr.read(url=attrs['icon_url']).count(): - icon_contr.create(**{'url': attrs['icon_url']}) + if not icon_contr.read(url=attrs["icon_url"]).count(): + icon_contr.create(**{"url": attrs["icon_url"]}) def create(self, **attrs): self._ensure_icon(attrs) @@ -87,12 +92,14 @@ class FeedController(AbstractController): def update(self, filters, attrs): from .article import ArticleController + self._ensure_icon(attrs) - if 'category_id' in attrs and attrs['category_id'] == 0: - del attrs['category_id'] - elif 'category_id' in attrs: + if "category_id" in attrs and attrs["category_id"] == 0: + del attrs["category_id"] + elif "category_id" in attrs: art_contr = ArticleController(self.user_id) for feed in self.read(**filters): - art_contr.update({'feed_id': feed.id}, - {'category_id': attrs['category_id']}) + art_contr.update( + {"feed_id": feed.id}, {"category_id": attrs["category_id"]} + ) return super().update(filters, attrs) diff --git a/newspipe/web/controllers/icon.py b/newspipe/web/controllers/icon.py index 07c4a4ef..de86b52f 100644 --- a/newspipe/web/controllers/icon.py +++ b/newspipe/web/controllers/icon.py @@ -9,11 +9,15 @@ class IconController(AbstractController): _user_id_key = None def _build_from_url(self, attrs): - if 'url' in attrs and 'content' not in attrs: - resp = requests.get(attrs['url'], verify=False) - attrs.update({'url': resp.url, - 'mimetype': resp.headers.get('content-type', None), - 'content': base64.b64encode(resp.content).decode('utf8')}) + if "url" in attrs and "content" not in attrs: + resp = requests.get(attrs["url"], verify=False) + attrs.update( + { + "url": resp.url, + "mimetype": resp.headers.get("content-type", None), + "content": base64.b64encode(resp.content).decode("utf8"), + } + ) return attrs def create(self, **attrs): diff --git a/newspipe/web/controllers/user.py b/newspipe/web/controllers/user.py index 6ab04d44..71eb7d08 100644 --- a/newspipe/web/controllers/user.py +++ b/newspipe/web/controllers/user.py @@ -8,13 +8,13 @@ logger = logging.getLogger(__name__) class UserController(AbstractController): _db_cls = User - _user_id_key = 'id' + _user_id_key = "id" def _handle_password(self, attrs): - if attrs.get('password'): - attrs['pwdhash'] = generate_password_hash(attrs.pop('password')) - elif 'password' in attrs: - del attrs['password'] + if attrs.get("password"): + attrs["pwdhash"] = generate_password_hash(attrs.pop("password")) + elif "password" in attrs: + del attrs["password"] def check_password(self, user, password): return check_password_hash(user.pwdhash, password) diff --git a/newspipe/web/decorators.py b/newspipe/web/decorators.py index 3835f646..8569a024 100644 --- a/newspipe/web/decorators.py +++ b/newspipe/web/decorators.py @@ -13,9 +13,11 @@ def async_maker(f): indexing the database) in background. This prevent the server to freeze. """ + def wrapper(*args, **kwargs): thr = Thread(target=f, args=args, kwargs=kwargs) thr.start() + return wrapper @@ -24,4 +26,5 @@ def pyagg_default_decorator(func): @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) + return wrapper diff --git a/newspipe/web/forms.py b/newspipe/web/forms.py index 7b1893e2..046ce1ec 100644 --- a/newspipe/web/forms.py +++ b/newspipe/web/forms.py @@ -30,8 +30,17 @@ from flask import flash, url_for, redirect from flask_wtf import FlaskForm from flask_babel import lazy_gettext from werkzeug.exceptions import NotFound -from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \ - SubmitField, IntegerField, SelectField, validators, HiddenField +from wtforms import ( + TextField, + TextAreaField, + PasswordField, + BooleanField, + SubmitField, + IntegerField, + SelectField, + validators, + HiddenField, +) from wtforms.fields.html5 import EmailField, URLField from lib import misc_utils @@ -43,27 +52,44 @@ class SignupForm(FlaskForm): """ Sign up form (registration to newspipe). """ - nickname = TextField(lazy_gettext("Nickname"), - [validators.Required(lazy_gettext("Please enter your nickname."))]) - email = EmailField(lazy_gettext("Email"), - [validators.Length(min=6, max=35), - validators.Required( - lazy_gettext("Please enter your email address (only for account activation, won't be stored)."))]) - password = PasswordField(lazy_gettext("Password"), - [validators.Required(lazy_gettext("Please enter a password.")), - validators.Length(min=6, max=100)]) + + nickname = TextField( + lazy_gettext("Nickname"), + [validators.Required(lazy_gettext("Please enter your nickname."))], + ) + email = EmailField( + lazy_gettext("Email"), + [ + validators.Length(min=6, max=35), + validators.Required( + lazy_gettext( + "Please enter your email address (only for account activation, won't be stored)." + ) + ), + ], + ) + password = PasswordField( + lazy_gettext("Password"), + [ + validators.Required(lazy_gettext("Please enter a password.")), + validators.Length(min=6, max=100), + ], + ) submit = SubmitField(lazy_gettext("Sign up")) def validate(self): ucontr = UserController() validated = super().validate() if ucontr.read(nickname=self.nickname.data).count(): - self.nickname.errors.append('Nickname already taken') + self.nickname.errors.append("Nickname already taken") validated = False if self.nickname.data != User.make_valid_nickname(self.nickname.data): - self.nickname.errors.append(lazy_gettext( - 'This nickname has invalid characters. ' - 'Please use letters, numbers, dots and underscores only.')) + self.nickname.errors.append( + lazy_gettext( + "This nickname has invalid characters. " + "Please use letters, numbers, dots and underscores only." + ) + ) validated = False return validated @@ -72,14 +98,15 @@ class RedirectForm(FlaskForm): """ Secure back redirects with WTForms. """ + next = HiddenField() def __init__(self, *args, **kwargs): FlaskForm.__init__(self, *args, **kwargs) if not self.next.data: - self.next.data = misc_utils.get_redirect_target() or '' + self.next.data = misc_utils.get_redirect_target() or "" - def redirect(self, endpoint='home', **values): + def redirect(self, endpoint="home", **values): if misc_utils.is_safe_url(self.next.data): return redirect(self.next.data) target = misc_utils.get_redirect_target() @@ -90,13 +117,21 @@ class SigninForm(RedirectForm): """ Sign in form (connection to newspipe). """ - nickmane = TextField("Nickname", - [validators.Length(min=3, max=35), - validators.Required( - lazy_gettext("Please enter your nickname."))]) - password = PasswordField(lazy_gettext('Password'), - [validators.Required(lazy_gettext("Please enter a password.")), - validators.Length(min=6, max=100)]) + + nickmane = TextField( + "Nickname", + [ + validators.Length(min=3, max=35), + validators.Required(lazy_gettext("Please enter your nickname.")), + ], + ) + password = PasswordField( + lazy_gettext("Password"), + [ + validators.Required(lazy_gettext("Please enter a password.")), + validators.Length(min=6, max=100), + ], + ) submit = SubmitField(lazy_gettext("Log In")) def __init__(self, *args, **kwargs): @@ -109,15 +144,14 @@ class SigninForm(RedirectForm): try: user = ucontr.get(nickname=self.nickmane.data) except NotFound: - self.nickmane.errors.append( - 'Wrong nickname') + self.nickmane.errors.append("Wrong nickname") validated = False else: if not user.is_active: - self.nickmane.errors.append('Account not active') + self.nickmane.errors.append("Account not active") validated = False if not ucontr.check_password(user, self.password.data): - self.password.errors.append('Wrong password') + self.password.errors.append("Wrong password") validated = False self.user = user return validated @@ -127,19 +161,24 @@ class UserForm(FlaskForm): """ Create or edit a user (for the administrator). """ - nickname = TextField(lazy_gettext("Nickname"), - [validators.Required(lazy_gettext("Please enter your nickname."))]) + + nickname = TextField( + lazy_gettext("Nickname"), + [validators.Required(lazy_gettext("Please enter your nickname."))], + ) password = PasswordField(lazy_gettext("Password")) - automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), - default=True) + automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True) submit = SubmitField(lazy_gettext("Save")) def validate(self): validated = super(UserForm, self).validate() if self.nickname.data != User.make_valid_nickname(self.nickname.data): - self.nickname.errors.append(lazy_gettext( - 'This nickname has invalid characters. ' - 'Please use letters, numbers, dots and underscores only.')) + self.nickname.errors.append( + lazy_gettext( + "This nickname has invalid characters. " + "Please use letters, numbers, dots and underscores only." + ) + ) validated = False return validated @@ -148,17 +187,18 @@ class ProfileForm(FlaskForm): """ Edit user information. """ - nickname = TextField(lazy_gettext("Nickname"), - [validators.Required(lazy_gettext("Please enter your nickname."))]) + + nickname = TextField( + lazy_gettext("Nickname"), + [validators.Required(lazy_gettext("Please enter your nickname."))], + ) password = PasswordField(lazy_gettext("Password")) password_conf = PasswordField(lazy_gettext("Password Confirmation")) - automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), - default=True) + automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True) bio = TextAreaField(lazy_gettext("Bio")) webpage = URLField(lazy_gettext("Webpage")) twitter = URLField(lazy_gettext("Twitter")) - is_public_profile = BooleanField(lazy_gettext("Public profile"), - default=True) + is_public_profile = BooleanField(lazy_gettext("Public profile"), default=True) submit = SubmitField(lazy_gettext("Save")) def validate(self): @@ -169,28 +209,34 @@ class ProfileForm(FlaskForm): self.password_conf.errors.append(message) validated = False if self.nickname.data != User.make_valid_nickname(self.nickname.data): - self.nickname.errors.append(lazy_gettext('This nickname has ' - 'invalid characters. Please use letters, numbers, dots and' - ' underscores only.')) + self.nickname.errors.append( + lazy_gettext( + "This nickname has " + "invalid characters. Please use letters, numbers, dots and" + " underscores only." + ) + ) validated = False return validated class AddFeedForm(FlaskForm): title = TextField(lazy_gettext("Title"), [validators.Optional()]) - link = TextField(lazy_gettext("Feed link"), - [validators.Required(lazy_gettext("Please enter the URL."))]) + link = TextField( + lazy_gettext("Feed link"), + [validators.Required(lazy_gettext("Please enter the URL."))], + ) site_link = TextField(lazy_gettext("Site link"), [validators.Optional()]) enabled = BooleanField(lazy_gettext("Check for updates"), default=True) submit = SubmitField(lazy_gettext("Save")) - category_id = SelectField(lazy_gettext("Category of the feed"), - [validators.Optional()]) + category_id = SelectField( + lazy_gettext("Category of the feed"), [validators.Optional()] + ) private = BooleanField(lazy_gettext("Private"), default=False) def set_category_choices(self, categories): - self.category_id.choices = [('0', 'No Category')] - self.category_id.choices += [(str(cat.id), cat.name) - for cat in categories] + self.category_id.choices = [("0", "No Category")] + self.category_id.choices += [(str(cat.id), cat.name) for cat in categories] class CategoryForm(FlaskForm): @@ -199,13 +245,13 @@ class CategoryForm(FlaskForm): class BookmarkForm(FlaskForm): - href = TextField(lazy_gettext("URL"), - [validators.Required( - lazy_gettext("Please enter an URL."))]) - title = TextField(lazy_gettext("Title"), - [validators.Length(min=0, max=100)]) - description = TextField(lazy_gettext("Description"), - [validators.Length(min=0, max=500)]) + href = TextField( + lazy_gettext("URL"), [validators.Required(lazy_gettext("Please enter an URL."))] + ) + title = TextField(lazy_gettext("Title"), [validators.Length(min=0, max=100)]) + description = TextField( + lazy_gettext("Description"), [validators.Length(min=0, max=500)] + ) tags = TextField(lazy_gettext("Tags")) to_read = BooleanField(lazy_gettext("To read"), default=False) shared = BooleanField(lazy_gettext("Shared"), default=False) @@ -213,8 +259,12 @@ class BookmarkForm(FlaskForm): class InformationMessageForm(FlaskForm): - subject = TextField(lazy_gettext("Subject"), - [validators.Required(lazy_gettext("Please enter a subject."))]) - message = TextAreaField(lazy_gettext("Message"), - [validators.Required(lazy_gettext("Please enter a content."))]) + subject = TextField( + lazy_gettext("Subject"), + [validators.Required(lazy_gettext("Please enter a subject."))], + ) + message = TextAreaField( + lazy_gettext("Message"), + [validators.Required(lazy_gettext("Please enter a content."))], + ) submit = SubmitField(lazy_gettext("Send")) diff --git a/newspipe/web/lib/user_utils.py b/newspipe/web/lib/user_utils.py index f78a6ed6..84b1c75c 100644 --- a/newspipe/web/lib/user_utils.py +++ b/newspipe/web/lib/user_utils.py @@ -1,22 +1,20 @@ - - from itsdangerous import URLSafeTimedSerializer import conf from bootstrap import application def generate_confirmation_token(nickname): - serializer = URLSafeTimedSerializer(application.config['SECRET_KEY']) - return serializer.dumps(nickname, salt=application.config['SECURITY_PASSWORD_SALT']) + serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"]) + return serializer.dumps(nickname, salt=application.config["SECURITY_PASSWORD_SALT"]) def confirm_token(token): - serializer = URLSafeTimedSerializer(application.config['SECRET_KEY']) + serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"]) try: nickname = serializer.loads( token, - salt=application.config['SECURITY_PASSWORD_SALT'], - max_age=conf.TOKEN_VALIDITY_PERIOD + salt=application.config["SECURITY_PASSWORD_SALT"], + max_age=conf.TOKEN_VALIDITY_PERIOD, ) except: return False diff --git a/newspipe/web/lib/view_utils.py b/newspipe/web/lib/view_utils.py index 1d8c6aed..218ebb4c 100644 --- a/newspipe/web/lib/view_utils.py +++ b/newspipe/web/lib/view_utils.py @@ -15,12 +15,14 @@ def etag_match(func): headers = {} else: return response - if request.headers.get('if-none-match') == etag: + if request.headers.get("if-none-match") == etag: response = Response(status=304) - response.headers['Cache-Control'] \ - = headers.get('Cache-Control', 'pragma: no-cache') + response.headers["Cache-Control"] = headers.get( + "Cache-Control", "pragma: no-cache" + ) elif not isinstance(response, Response): response = make_response(response) - response.headers['etag'] = etag + response.headers["etag"] = etag return response + return wrapper diff --git a/newspipe/web/models/__init__.py b/newspipe/web/models/__init__.py index bfb1368c..09249368 100644 --- a/newspipe/web/models/__init__.py +++ b/newspipe/web/models/__init__.py @@ -36,18 +36,29 @@ from .tag import BookmarkTag from .tag import ArticleTag from .bookmark import Bookmark -__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category', - 'Bookmark', 'ArticleTag', 'BookmarkTag'] +__all__ = [ + "Feed", + "Role", + "User", + "Article", + "Icon", + "Category", + "Bookmark", + "ArticleTag", + "BookmarkTag", +] import os from sqlalchemy.engine import reflection from sqlalchemy.schema import ( - MetaData, - Table, - DropTable, - ForeignKeyConstraint, - DropConstraint) + MetaData, + Table, + DropTable, + ForeignKeyConstraint, + DropConstraint, +) + def db_empty(db): "Will drop every datas stocked in db." @@ -71,9 +82,9 @@ def db_empty(db): for table_name in inspector.get_table_names(): fks = [] for fk in inspector.get_foreign_keys(table_name): - if not fk['name']: + if not fk["name"]: continue - fks.append(ForeignKeyConstraint((), (), name=fk['name'])) + fks.append(ForeignKeyConstraint((), (), name=fk["name"])) t = Table(table_name, metadata, *fks) tbs.append(t) all_fks.extend(fks) diff --git a/newspipe/web/models/article.py b/newspipe/web/models/article.py index d55e59c1..c826a3c9 100644 --- a/newspipe/web/models/article.py +++ b/newspipe/web/models/article.py @@ -48,40 +48,54 @@ class Article(db.Model, RightMixin): retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow) # foreign keys - user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) - feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id')) - category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) + feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id")) + category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) # relationships - tag_objs = db.relationship('ArticleTag', back_populates='article', - cascade='all,delete-orphan', - lazy=False, - foreign_keys='[ArticleTag.article_id]') - tags = association_proxy('tag_objs', 'text') + tag_objs = db.relationship( + "ArticleTag", + back_populates="article", + cascade="all,delete-orphan", + lazy=False, + foreign_keys="[ArticleTag.article_id]", + ) + tags = association_proxy("tag_objs", "text") # indexes - #__table_args__ = ( + # __table_args__ = ( # Index('user_id'), # Index('user_id', 'category_id'), # Index('user_id', 'feed_id'), # Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id) - #) + # ) # api whitelists @staticmethod def _fields_base_write(): - return {'readed', 'like', 'feed_id', 'category_id'} + return {"readed", "like", "feed_id", "category_id"} @staticmethod def _fields_base_read(): - return {'id', 'entry_id', 'link', 'title', 'content', 'date', - 'retrieved_date', 'user_id', 'tags'} + return { + "id", + "entry_id", + "link", + "title", + "content", + "date", + "retrieved_date", + "user_id", + "tags", + } @staticmethod def _fields_api_write(): - return {'tags'} + return {"tags"} def __repr__(self): - return "<Article(id=%d, entry_id=%s, title=%r, " \ - "date=%r, retrieved_date=%r)>" % (self.id, self.entry_id, - self.title, self.date, self.retrieved_date) + return ( + "<Article(id=%d, entry_id=%s, title=%r, " + "date=%r, retrieved_date=%r)>" + % (self.id, self.entry_id, self.title, self.date, self.retrieved_date) + ) diff --git a/newspipe/web/models/bookmark.py b/newspipe/web/models/bookmark.py index eb6b73e3..c557225e 100644 --- a/newspipe/web/models/bookmark.py +++ b/newspipe/web/models/bookmark.py @@ -40,6 +40,7 @@ class Bookmark(db.Model, RightMixin): """ Represent a bookmark. """ + id = db.Column(db.Integer(), primary_key=True) href = db.Column(db.String(), default="") title = db.Column(db.String(), default="") @@ -47,22 +48,25 @@ class Bookmark(db.Model, RightMixin): shared = db.Column(db.Boolean(), default=False) to_read = db.Column(db.Boolean(), default=False) time = db.Column(db.DateTime(), default=datetime.utcnow) - user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) # relationships - tags = db.relationship(BookmarkTag, backref='of_bookmark', lazy='dynamic', - cascade='all,delete-orphan', - order_by=desc(BookmarkTag.text)) - tags_proxy = association_proxy('tags', 'text') - + tags = db.relationship( + BookmarkTag, + backref="of_bookmark", + lazy="dynamic", + cascade="all,delete-orphan", + order_by=desc(BookmarkTag.text), + ) + tags_proxy = association_proxy("tags", "text") - @validates('description') + @validates("description") def validates_title(self, key, value): return str(value).strip() - @validates('extended') + @validates("extended") def validates_description(self, key, value): return str(value).strip() def __repr__(self): - return '<Bookmark %r>' % (self.href) + return "<Bookmark %r>" % (self.href) diff --git a/newspipe/web/models/category.py b/newspipe/web/models/category.py index 2da7809a..bb47ce45 100644 --- a/newspipe/web/models/category.py +++ b/newspipe/web/models/category.py @@ -11,19 +11,18 @@ class Category(db.Model, RightMixin): name = db.Column(db.String()) # relationships - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - feeds = db.relationship('Feed', cascade='all,delete-orphan') - articles = db.relationship('Article', - cascade='all,delete-orphan') + user_id = db.Column(db.Integer, db.ForeignKey("user.id")) + feeds = db.relationship("Feed", cascade="all,delete-orphan") + articles = db.relationship("Article", cascade="all,delete-orphan") # index - idx_category_uid = Index('user_id') + idx_category_uid = Index("user_id") # api whitelists @staticmethod def _fields_base_read(): - return {'id', 'user_id'} + return {"id", "user_id"} @staticmethod def _fields_base_write(): - return {'name'} + return {"name"} diff --git a/newspipe/web/models/feed.py b/newspipe/web/models/feed.py index fc0b64cb..f440a39c 100644 --- a/newspipe/web/models/feed.py +++ b/newspipe/web/models/feed.py @@ -38,6 +38,7 @@ class Feed(db.Model, RightMixin): """ Represent a feed. """ + id = db.Column(db.Integer(), primary_key=True) title = db.Column(db.String(), default="") description = db.Column(db.String(), default="FR") @@ -58,34 +59,47 @@ class Feed(db.Model, RightMixin): error_count = db.Column(db.Integer(), default=0) # relationship - icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None) - user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) - category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) - articles = db.relationship(Article, backref='source', lazy='dynamic', - cascade='all,delete-orphan', - order_by=desc(Article.date)) + icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None) + user_id = db.Column(db.Integer(), db.ForeignKey("user.id")) + category_id = db.Column(db.Integer(), db.ForeignKey("category.id")) + articles = db.relationship( + Article, + backref="source", + lazy="dynamic", + cascade="all,delete-orphan", + order_by=desc(Article.date), + ) # index - idx_feed_uid_cid = Index('user_id', 'category_id') - idx_feed_uid = Index('user_id') + idx_feed_uid_cid = Index("user_id", "category_id") + idx_feed_uid = Index("user_id") - # api whitelists + # api whitelists @staticmethod def _fields_base_write(): - return {'title', 'description', 'link', 'site_link', 'enabled', - 'filters', 'last_error', 'error_count', 'category_id'} + return { + "title", + "description", + "link", + "site_link", + "enabled", + "filters", + "last_error", + "error_count", + "category_id", + } @staticmethod def _fields_base_read(): - return {'id', 'user_id', 'icon_url', 'last_retrieved'} + return {"id", "user_id", "icon_url", "last_retrieved"} - @validates('title') + @validates("title") def validates_title(self, key, value): return str(value).strip() - @validates('description') + @validates("description") def validates_description(self, key, value): return str(value).strip() def __repr__(self): - return '<Feed %r>' % (self.title) + return "<Feed %r>" % (self.title) diff --git a/newspipe/web/models/right_mixin.py b/newspipe/web/models/right_mixin.py index 1c316f95..670beafa 100644 --- a/newspipe/web/models/right_mixin.py +++ b/newspipe/web/models/right_mixin.py @@ -2,14 +2,13 @@ from sqlalchemy.ext.associationproxy import _AssociationList class RightMixin: - @staticmethod def _fields_base_write(): return set() @staticmethod def _fields_base_read(): - return set(['id']) + return set(["id"]) @staticmethod def _fields_api_write(): @@ -17,7 +16,7 @@ class RightMixin: @staticmethod def _fields_api_read(): - return set(['id']) + return set(["id"]) @classmethod def fields_base_write(cls): @@ -36,26 +35,28 @@ class RightMixin: return cls.fields_base_read().union(cls._fields_api_read()) def __getitem__(self, key): - if not hasattr(self, '__dump__'): + if not hasattr(self, "__dump__"): self.__dump__ = {} return self.__dump__.get(key) def __setitem__(self, key, value): - if not hasattr(self, '__dump__'): + if not hasattr(self, "__dump__"): self.__dump__ = {} self.__dump__[key] = value - def dump(self, role='admin'): - if role == 'admin': - dico = {k: getattr(self, k) - for k in set(self.__table__.columns.keys()) - .union(self.fields_api_read()) - .union(self.fields_base_read())} - elif role == 'api': + def dump(self, role="admin"): + if role == "admin": + dico = { + k: getattr(self, k) + for k in set(self.__table__.columns.keys()) + .union(self.fields_api_read()) + .union(self.fields_base_read()) + } + elif role == "api": dico = {k: getattr(self, k) for k in self.fields_api_read()} else: dico = {k: getattr(self, k) for k in self.fields_base_read()} - if hasattr(self, '__dump__'): + if hasattr(self, "__dump__"): dico.update(self.__dump__) for key, value in dico.items(): # preventing association proxy to die if isinstance(value, _AssociationList): diff --git a/newspipe/web/models/role.py b/newspipe/web/models/role.py index 0a2ecd4a..15a08b73 100644 --- a/newspipe/web/models/role.py +++ b/newspipe/web/models/role.py @@ -33,7 +33,8 @@ class Role(db.Model): """ Represent a role. """ + id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(), unique=True) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + user_id = db.Column(db.Integer, db.ForeignKey("user.id")) diff --git a/newspipe/web/models/tag.py b/newspipe/web/models/tag.py index 76467c0b..9bd8afc5 100644 --- a/newspipe/web/models/tag.py +++ b/newspipe/web/models/tag.py @@ -8,12 +8,14 @@ class ArticleTag(db.Model): text = db.Column(db.String, primary_key=True, unique=False) # foreign keys - article_id = db.Column(db.Integer, db.ForeignKey('article.id', ondelete='CASCADE'), - primary_key=True) + article_id = db.Column( + db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True + ) # relationships - article = db.relationship('Article', back_populates='tag_objs', - foreign_keys=[article_id]) + article = db.relationship( + "Article", back_populates="tag_objs", foreign_keys=[article_id] + ) def __init__(self, text): self.text = text @@ -24,12 +26,18 @@ class BookmarkTag(db.Model): text = db.Column(db.String, unique=False) # foreign keys - user_id = db.Column(db.Integer, db.ForeignKey('user.id', ondelete='CASCADE')) - bookmark_id = db.Column(db.Integer, db.ForeignKey('bookmark.id', ondelete='CASCADE')) + user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE")) + bookmark_id = db.Column( + db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE") + ) # relationships - bookmark = db.relationship('Bookmark', back_populates='tags', - cascade="all,delete", foreign_keys=[bookmark_id]) + bookmark = db.relationship( + "Bookmark", + back_populates="tags", + cascade="all,delete", + foreign_keys=[bookmark_id], + ) # def __init__(self, text, user_id): # self.text = text diff --git a/newspipe/web/models/user.py b/newspipe/web/models/user.py index 4d65c3c5..a088c74e 100644 --- a/newspipe/web/models/user.py +++ b/newspipe/web/models/user.py @@ -44,6 +44,7 @@ class User(db.Model, UserMixin, RightMixin): """ Represent a user. """ + id = db.Column(db.Integer, primary_key=True) nickname = db.Column(db.String(), unique=True) pwdhash = db.Column(db.String()) @@ -64,29 +65,34 @@ class User(db.Model, UserMixin, RightMixin): is_api = db.Column(db.Boolean(), default=False) # relationships - categories = db.relationship('Category', backref='user', - cascade='all, delete-orphan', - foreign_keys=[Category.user_id]) - feeds = db.relationship('Feed', backref='user', - cascade='all, delete-orphan', - foreign_keys=[Feed.user_id]) + categories = db.relationship( + "Category", + backref="user", + cascade="all, delete-orphan", + foreign_keys=[Category.user_id], + ) + feeds = db.relationship( + "Feed", + backref="user", + cascade="all, delete-orphan", + foreign_keys=[Feed.user_id], + ) @staticmethod def _fields_base_write(): - return {'login', 'password'} + return {"login", "password"} @staticmethod def _fields_base_read(): - return {'date_created', 'last_connection'} + return {"date_created", "last_connection"} @staticmethod def make_valid_nickname(nickname): - return re.sub('[^a-zA-Z0-9_\.]', '', nickname) + return re.sub("[^a-zA-Z0-9_\.]", "", nickname) - @validates('bio') + @validates("bio") def validates_bio(self, key, value): - assert len(value) <= 5000, \ - AssertionError("maximum length for bio: 5000") + assert len(value) <= 5000, AssertionError("maximum length for bio: 5000") return value.strip() def get_id(self): @@ -105,4 +111,4 @@ class User(db.Model, UserMixin, RightMixin): return self.id == other.id def __repr__(self): - return '<User %r>' % (self.nickname) + return "<User %r>" % (self.nickname) diff --git a/newspipe/web/views/__init__.py b/newspipe/web/views/__init__.py index 41bb52f3..b8b415f9 100644 --- a/newspipe/web/views/__init__.py +++ b/newspipe/web/views/__init__.py @@ -8,10 +8,25 @@ from web.views.admin import admin_bp from web.views.user import user_bp, users_bp from web.views.bookmark import bookmark_bp, bookmarks_bp -__all__ = ['views', 'home', 'session_mgmt', 'v2', 'v3', - 'article_bp', 'articles_bp', 'feed_bp', 'feeds_bp', - 'category_bp', 'categories_bp', 'icon_bp', - 'admin_bp', 'user_bp', 'users_bp', 'bookmark_bp', 'bookmarks_bp'] +__all__ = [ + "views", + "home", + "session_mgmt", + "v2", + "v3", + "article_bp", + "articles_bp", + "feed_bp", + "feeds_bp", + "category_bp", + "categories_bp", + "icon_bp", + "admin_bp", + "user_bp", + "users_bp", + "bookmark_bp", + "bookmarks_bp", +] import conf from flask import request diff --git a/newspipe/web/views/admin.py b/newspipe/web/views/admin.py index 73b2b668..fe1b389b 100644 --- a/newspipe/web/views/admin.py +++ b/newspipe/web/views/admin.py @@ -1,5 +1,5 @@ from datetime import datetime -from flask import (Blueprint, render_template, redirect, flash, url_for) +from flask import Blueprint, render_template, redirect, flash, url_for from flask_babel import gettext, format_timedelta from flask_login import login_required, current_user @@ -8,41 +8,45 @@ from web.views.common import admin_permission from web.controllers import UserController from web.forms import InformationMessageForm, UserForm -admin_bp = Blueprint('admin', __name__, url_prefix='/admin') +admin_bp = Blueprint("admin", __name__, url_prefix="/admin") -@admin_bp.route('/dashboard', methods=['GET', 'POST']) +@admin_bp.route("/dashboard", methods=["GET", "POST"]) @login_required @admin_permission.require(http_exception=403) def dashboard(): last_cons, now = {}, datetime.utcnow() - users = list(UserController().read().order_by('id')) + users = list(UserController().read().order_by("id")) form = InformationMessageForm() for user in users: last_cons[user.id] = format_timedelta(now - user.last_seen) - return render_template('admin/dashboard.html', now=datetime.utcnow(), - last_cons=last_cons, users=users, current_user=current_user, - form=form) - - -@admin_bp.route('/user/create', methods=['GET']) -@admin_bp.route('/user/edit/<int:user_id>', methods=['GET']) + return render_template( + "admin/dashboard.html", + now=datetime.utcnow(), + last_cons=last_cons, + users=users, + current_user=current_user, + form=form, + ) + + +@admin_bp.route("/user/create", methods=["GET"]) +@admin_bp.route("/user/edit/<int:user_id>", methods=["GET"]) @login_required @admin_permission.require(http_exception=403) def user_form(user_id=None): if user_id is not None: user = UserController().get(id=user_id) form = UserForm(obj=user) - message = gettext('Edit the user <i>%(nick)s</i>', nick=user.nickname) + message = gettext("Edit the user <i>%(nick)s</i>", nick=user.nickname) else: form = UserForm() - message = gettext('Add a new user') - return render_template('/admin/create_user.html', - form=form, message=message) + message = gettext("Add a new user") + return render_template("/admin/create_user.html", form=form, message=message) -@admin_bp.route('/user/create', methods=['POST']) -@admin_bp.route('/user/edit/<int:user_id>', methods=['POST']) +@admin_bp.route("/user/create", methods=["POST"]) +@admin_bp.route("/user/edit/<int:user_id>", methods=["POST"]) @login_required @admin_permission.require(http_exception=403) def process_user_form(user_id=None): @@ -53,31 +57,42 @@ def process_user_form(user_id=None): user_contr = UserController() if not form.validate(): - return render_template('/admin/create_user.html', form=form, - message=gettext('Some errors were found')) + return render_template( + "/admin/create_user.html", + form=form, + message=gettext("Some errors were found"), + ) if user_id is not None: # Edit a user - user_contr.update({'id': user_id}, - {'nickname': form.nickname.data, - 'password': form.password.data, - 'automatic_crawling': form.automatic_crawling.data}) + user_contr.update( + {"id": user_id}, + { + "nickname": form.nickname.data, + "password": form.password.data, + "automatic_crawling": form.automatic_crawling.data, + }, + ) user = user_contr.get(id=user_id) - flash(gettext('User %(nick)s successfully updated', - nick=user.nickname), 'success') + flash( + gettext("User %(nick)s successfully updated", nick=user.nickname), "success" + ) else: # Create a new user (by the admin) - user = user_contr.create(nickname=form.nickname.data, - password=form.password.data, - automatic_crawling=form.automatic_crawling.data, - is_admin=False, - is_active=True) - flash(gettext('User %(nick)s successfully created', - nick=user.nickname), 'success') - return redirect(url_for('admin.user_form', user_id=user.id)) + user = user_contr.create( + nickname=form.nickname.data, + password=form.password.data, + automatic_crawling=form.automatic_crawling.data, + is_admin=False, + is_active=True, + ) + flash( + gettext("User %(nick)s successfully created", nick=user.nickname), "success" + ) + return redirect(url_for("admin.user_form", user_id=user.id)) -@admin_bp.route('/delete_user/<int:user_id>', methods=['GET']) +@admin_bp.route("/delete_user/<int:user_id>", methods=["GET"]) @login_required @admin_permission.require(http_exception=403) def delete_user(user_id=None): @@ -86,16 +101,21 @@ def delete_user(user_id=None): """ try: user = UserController().delete(user_id) - flash(gettext('User %(nick)s successfully deleted', - nick=user.nickname), 'success') + flash( + gettext("User %(nick)s successfully deleted", nick=user.nickname), "success" + ) except Exception as error: flash( - gettext('An error occurred while trying to delete a user: %(error)s', - error=error), 'danger') - return redirect(url_for('admin.dashboard')) + gettext( + "An error occurred while trying to delete a user: %(error)s", + error=error, + ), + "danger", + ) + return redirect(url_for("admin.dashboard")) -@admin_bp.route('/toggle_user/<int:user_id>', methods=['GET']) +@admin_bp.route("/toggle_user/<int:user_id>", methods=["GET"]) @login_required @admin_permission.require() def toggle_user(user_id=None): @@ -104,16 +124,18 @@ def toggle_user(user_id=None): """ ucontr = UserController() user = ucontr.get(id=user_id) - user_changed = ucontr.update({'id': user_id}, - {'is_active': not user.is_active}) + user_changed = ucontr.update({"id": user_id}, {"is_active": not user.is_active}) if not user_changed: - flash(gettext('This user does not exist.'), 'danger') - return redirect(url_for('admin.dashboard')) + flash(gettext("This user does not exist."), "danger") + return redirect(url_for("admin.dashboard")) else: - act_txt = 'activated' if user.is_active else 'desactivated' - message = gettext('User %(nickname)s successfully %(is_active)s', - nickname=user.nickname, is_active=act_txt) - flash(message, 'success') - return redirect(url_for('admin.dashboard')) + act_txt = "activated" if user.is_active else "desactivated" + message = gettext( + "User %(nickname)s successfully %(is_active)s", + nickname=user.nickname, + is_active=act_txt, + ) + flash(message, "success") + return redirect(url_for("admin.dashboard")) diff --git a/newspipe/web/views/api/v2/__init__.py b/newspipe/web/views/api/v2/__init__.py index 46760261..ef587e72 100644 --- a/newspipe/web/views/api/v2/__init__.py +++ b/newspipe/web/views/api/v2/__init__.py @@ -1,3 +1,3 @@ from web.views.api.v2 import article, feed, category -__all__ = ['article', 'feed', 'category'] +__all__ = ["article", "feed", "category"] diff --git a/newspipe/web/views/api/v2/article.py b/newspipe/web/views/api/v2/article.py index 2be286c6..8da6c6dd 100644 --- a/newspipe/web/views/api/v2/article.py +++ b/newspipe/web/views/api/v2/article.py @@ -6,8 +6,12 @@ from flask_restful import Api from web.views.common import api_permission from web.controllers import ArticleController -from web.views.api.v2.common import (PyAggAbstractResource, - PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti) +from web.views.api.v2.common import ( + PyAggAbstractResource, + PyAggResourceNew, + PyAggResourceExisting, + PyAggResourceMulti, +) class ArticleNewAPI(PyAggResourceNew): @@ -24,30 +28,32 @@ class ArticlesAPI(PyAggResourceMulti): class ArticlesChallenge(PyAggAbstractResource): controller_cls = ArticleController - attrs = {'ids': {'type': list, 'default': []}} + attrs = {"ids": {"type": list, "default": []}} @api_permission.require(http_exception=403) def get(self): - parsed_args = self.reqparse_args(right='read') + parsed_args = self.reqparse_args(right="read") # collecting all attrs for casting purpose - attrs = self.controller_cls._get_attrs_desc('admin') - for id_dict in parsed_args['ids']: + attrs = self.controller_cls._get_attrs_desc("admin") + for id_dict in parsed_args["ids"]: keys_to_ignore = [] for key in id_dict: if key not in attrs: keys_to_ignore.append(key) - if issubclass(attrs[key]['type'], datetime): + if issubclass(attrs[key]["type"], datetime): id_dict[key] = dateutil.parser.parse(id_dict[key]) for key in keys_to_ignore: del id_dict[key] - result = list(self.controller.challenge(parsed_args['ids'])) + result = list(self.controller.challenge(parsed_args["ids"])) return result or None, 200 if result else 204 + api = Api(current_app, prefix=API_ROOT) -api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') -api.add_resource(ArticleAPI, '/article/<int:obj_id>', endpoint='article.json') -api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') -api.add_resource(ArticlesChallenge, '/articles/challenge', - endpoint='articles_challenge.json') +api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json") +api.add_resource(ArticleAPI, "/article/<int:obj_id>", endpoint="article.json") +api.add_resource(ArticlesAPI, "/articles", endpoint="articles.json") +api.add_resource( + ArticlesChallenge, "/articles/challenge", endpoint="articles_challenge.json" +) diff --git a/newspipe/web/views/api/v2/category.py b/newspipe/web/views/api/v2/category.py index 70fda1ea..a830624d 100644 --- a/newspipe/web/views/api/v2/category.py +++ b/newspipe/web/views/api/v2/category.py @@ -3,9 +3,11 @@ from flask import current_app from flask_restful import Api from web.controllers.category import CategoryController -from web.views.api.v2.common import (PyAggResourceNew, - PyAggResourceExisting, - PyAggResourceMulti) +from web.views.api.v2.common import ( + PyAggResourceNew, + PyAggResourceExisting, + PyAggResourceMulti, +) class CategoryNewAPI(PyAggResourceNew): @@ -21,7 +23,6 @@ class CategoriesAPI(PyAggResourceMulti): api = Api(current_app, prefix=API_ROOT) -api.add_resource(CategoryNewAPI, '/category', endpoint='category_new.json') -api.add_resource(CategoryAPI, '/category/<int:obj_id>', - endpoint='category.json') -api.add_resource(CategoriesAPI, '/categories', endpoint='categories.json') +api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json") +api.add_resource(CategoryAPI, "/category/<int:obj_id>", endpoint="category.json") +api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json") diff --git a/newspipe/web/views/api/v2/common.py b/newspipe/web/views/api/v2/common.py index 8a53d7e6..81248422 100644 --- a/newspipe/web/views/api/v2/common.py +++ b/newspipe/web/views/api/v2/common.py @@ -26,8 +26,12 @@ from flask import request from flask_restful import Resource, reqparse from flask_login import current_user -from web.views.common import admin_permission, api_permission, \ - login_user_bundle, jsonify +from web.views.common import ( + admin_permission, + api_permission, + login_user_bundle, + jsonify, +) from web.controllers import UserController logger = logging.getLogger(__name__) @@ -50,6 +54,7 @@ def authenticate(func): if current_user.is_authenticated: return func(*args, **kwargs) raise Unauthorized() + return wrapper @@ -64,8 +69,9 @@ class PyAggAbstractResource(Resource): return self.controller_cls() return self.controller_cls(current_user.id) - def reqparse_args(self, right, req=None, strict=False, default=True, - allow_empty=False): + def reqparse_args( + self, right, req=None, strict=False, default=True, allow_empty=False + ): """ strict: bool if True will throw 400 error if args are defined and not in request @@ -89,42 +95,41 @@ class PyAggAbstractResource(Resource): if self.attrs is not None: attrs = self.attrs elif admin_permission.can(): - attrs = self.controller_cls._get_attrs_desc('admin') + attrs = self.controller_cls._get_attrs_desc("admin") elif api_permission.can(): - attrs = self.controller_cls._get_attrs_desc('api', right) + attrs = self.controller_cls._get_attrs_desc("api", right) else: - attrs = self.controller_cls._get_attrs_desc('base', right) + attrs = self.controller_cls._get_attrs_desc("base", right) assert attrs, "No defined attrs for %s" % self.__class__.__name__ for attr_name, attr in attrs.items(): if not default and attr_name not in in_values: continue else: - parser.add_argument(attr_name, location='json', - default=in_values[attr_name]) + parser.add_argument( + attr_name, location="json", default=in_values[attr_name] + ) return parser.parse_args(req=request.args, strict=strict) class PyAggResourceNew(PyAggAbstractResource): - @api_permission.require(http_exception=403) def post(self): """Create a single new object""" - return self.controller.create(**self.reqparse_args(right='write')), 201 + return self.controller.create(**self.reqparse_args(right="write")), 201 class PyAggResourceExisting(PyAggAbstractResource): - def get(self, obj_id=None): """Retrieve a single object""" return self.controller.get(id=obj_id) def put(self, obj_id=None): """update an object, new attrs should be passed in the payload""" - args = self.reqparse_args(right='write', default=False) + args = self.reqparse_args(right="write", default=False) if not args: raise BadRequest() - return self.controller.update({'id': obj_id}, args), 200 + return self.controller.update({"id": obj_id}, args), 200 def delete(self, obj_id=None): """delete a object""" @@ -133,19 +138,18 @@ class PyAggResourceExisting(PyAggAbstractResource): class PyAggResourceMulti(PyAggAbstractResource): - def get(self): """retrieve several objects. filters can be set in the payload on the different fields of the object, and a limit can be set in there as well """ args = {} try: - limit = request.json.pop('limit', 10) - order_by = request.json.pop('order_by', None) + limit = request.json.pop("limit", 10) + order_by = request.json.pop("order_by", None) except Exception: - args = self.reqparse_args(right='read', default=False) - limit = request.args.get('limit', 10) - order_by = request.args.get('order_by', None) + args = self.reqparse_args(right="read", default=False) + limit = request.args.get("limit", 10) + order_by = request.args.get("order_by", None) query = self.controller.read(**args) if order_by: query = query.order_by(order_by) @@ -163,10 +167,11 @@ class PyAggResourceMulti(PyAggAbstractResource): class Proxy: pass + for attrs in request.json: try: Proxy.json = attrs - args = self.reqparse_args('write', req=Proxy, default=False) + args = self.reqparse_args("write", req=Proxy, default=False) obj = self.controller.create(**args) results.append(obj) except Exception as error: @@ -188,20 +193,21 @@ class PyAggResourceMulti(PyAggAbstractResource): class Proxy: pass + for obj_id, attrs in request.json: try: Proxy.json = attrs - args = self.reqparse_args('write', req=Proxy, default=False) - result = self.controller.update({'id': obj_id}, args) + args = self.reqparse_args("write", req=Proxy, default=False) + result = self.controller.update({"id": obj_id}, args) if result: - results.append('ok') + results.append("ok") else: - results.append('nok') + results.append("nok") except Exception as error: results.append(str(error)) - if results.count('ok') == 0: # all failed => 500 + if results.count("ok") == 0: # all failed => 500 status = 500 - elif results.count('ok') != len(results): # some failed => 206 + elif results.count("ok") != len(results): # some failed => 206 status = 206 return results, status @@ -212,11 +218,11 @@ class PyAggResourceMulti(PyAggAbstractResource): for obj_id in request.json: try: self.controller.delete(obj_id) - results.append('ok') + results.append("ok") except Exception as error: status = 206 results.append(error) # if no operation succeeded, it's not partial anymore, returning err 500 - if status == 206 and results.count('ok') == 0: + if status == 206 and results.count("ok") == 0: status = 500 return results, status diff --git a/newspipe/web/views/api/v2/feed.py b/newspipe/web/views/api/v2/feed.py index a0691277..1e4fabf2 100644 --- a/newspipe/web/views/api/v2/feed.py +++ b/newspipe/web/views/api/v2/feed.py @@ -3,14 +3,14 @@ from flask import current_app from flask_restful import Api from web.views.common import api_permission -from web.controllers.feed import (FeedController, - DEFAULT_MAX_ERROR, - DEFAULT_LIMIT) +from web.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from web.views.api.v2.common import PyAggAbstractResource, \ - PyAggResourceNew, \ - PyAggResourceExisting, \ - PyAggResourceMulti +from web.views.api.v2.common import ( + PyAggAbstractResource, + PyAggResourceNew, + PyAggResourceExisting, + PyAggResourceMulti, +) class FeedNewAPI(PyAggResourceNew): @@ -27,21 +27,21 @@ class FeedsAPI(PyAggResourceMulti): class FetchableFeedAPI(PyAggAbstractResource): controller_cls = FeedController - attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR}, - 'limit': {'type': int, 'default': DEFAULT_LIMIT}} + attrs = { + "max_error": {"type": int, "default": DEFAULT_MAX_ERROR}, + "limit": {"type": int, "default": DEFAULT_LIMIT}, + } @api_permission.require(http_exception=403) def get(self): - args = self.reqparse_args(right='read', allow_empty=True) - result = [feed for feed - in self.controller.list_fetchable(**args)] + args = self.reqparse_args(right="read", allow_empty=True) + result = [feed for feed in self.controller.list_fetchable(**args)] return result or None, 200 if result else 204 api = Api(current_app, prefix=API_ROOT) -api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') -api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json') -api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json') -api.add_resource(FetchableFeedAPI, '/feeds/fetchable', - endpoint='fetchable_feed.json') +api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json") +api.add_resource(FeedAPI, "/feed/<int:obj_id>", endpoint="feed.json") +api.add_resource(FeedsAPI, "/feeds", endpoint="feeds.json") +api.add_resource(FetchableFeedAPI, "/feeds/fetchable", endpoint="fetchable_feed.json") diff --git a/newspipe/web/views/api/v3/__init__.py b/newspipe/web/views/api/v3/__init__.py index 76aa1f19..5066b0b6 100644 --- a/newspipe/web/views/api/v3/__init__.py +++ b/newspipe/web/views/api/v3/__init__.py @@ -1,3 +1,3 @@ from web.views.api.v3 import article -__all__ = ['article'] +__all__ = ["article"] diff --git a/newspipe/web/views/api/v3/article.py b/newspipe/web/views/api/v3/article.py index 4cf35648..04d19552 100644 --- a/newspipe/web/views/api/v3/article.py +++ b/newspipe/web/views/api/v3/article.py @@ -35,6 +35,7 @@ from web.controllers import ArticleController, FeedController from web.views.api.v3.common import AbstractProcessor from web.views.api.v3.common import url_prefix, auth_func + class ArticleProcessor(AbstractProcessor): """Concrete processors for the Article Web service. """ @@ -43,7 +44,7 @@ class ArticleProcessor(AbstractProcessor): try: article = ArticleController(current_user.id).get(id=instance_id) except NotFound: - raise ProcessingException(description='No such article.', code=404) + raise ProcessingException(description="No such article.", code=404) self.is_authorized(current_user, article) def post_preprocessor(self, data=None, **kw): @@ -52,7 +53,7 @@ class ArticleProcessor(AbstractProcessor): try: feed = FeedController(current_user.id).get(id=data["feed_id"]) except NotFound: - raise ProcessingException(description='No such feed.', code=404) + raise ProcessingException(description="No such feed.", code=404) self.is_authorized(current_user, feed) data["category_id"] = feed.category_id @@ -61,24 +62,23 @@ class ArticleProcessor(AbstractProcessor): try: article = ArticleController(current_user.id).get(id=instance_id) except NotFound: - raise ProcessingException(description='No such article.', code=404) + raise ProcessingException(description="No such article.", code=404) self.is_authorized(current_user, article) + article_processor = ArticleProcessor() -blueprint_article = manager.create_api_blueprint(models.Article, - url_prefix=url_prefix, - methods=['GET', 'POST', 'PUT', 'DELETE'], - preprocessors=dict(GET_SINGLE=[auth_func, - article_processor.get_single_preprocessor], - GET_MANY=[auth_func, - article_processor.get_many_preprocessor], - POST=[auth_func, - article_processor.post_preprocessor], - PUT_SINGLE=[auth_func, - article_processor.put_single_preprocessor], - PUT_MANY=[auth_func, - article_processor.put_many_preprocessor], - DELETE=[auth_func, - article_processor.delete_preprocessor])) +blueprint_article = manager.create_api_blueprint( + models.Article, + url_prefix=url_prefix, + methods=["GET", "POST", "PUT", "DELETE"], + preprocessors=dict( + GET_SINGLE=[auth_func, article_processor.get_single_preprocessor], + GET_MANY=[auth_func, article_processor.get_many_preprocessor], + POST=[auth_func, article_processor.post_preprocessor], + PUT_SINGLE=[auth_func, article_processor.put_single_preprocessor], + PUT_MANY=[auth_func, article_processor.put_many_preprocessor], + DELETE=[auth_func, article_processor.delete_preprocessor], + ), +) application.register_blueprint(blueprint_article) diff --git a/newspipe/web/views/api/v3/common.py b/newspipe/web/views/api/v3/common.py index d5e94a3f..5467ee30 100644 --- a/newspipe/web/views/api/v3/common.py +++ b/newspipe/web/views/api/v3/common.py @@ -33,7 +33,8 @@ from werkzeug.exceptions import NotFound from web.controllers import ArticleController, UserController from web.views.common import login_user_bundle -url_prefix = '/api/v3' +url_prefix = "/api/v3" + def auth_func(*args, **kw): if request.authorization: @@ -41,24 +42,23 @@ def auth_func(*args, **kw): try: user = ucontr.get(nickname=request.authorization.username) except NotFound: - raise ProcessingException("Couldn't authenticate your user", - code=401) + raise ProcessingException("Couldn't authenticate your user", code=401) if not ucontr.check_password(user, request.authorization.password): - raise ProcessingException("Couldn't authenticate your user", - code=401) + raise ProcessingException("Couldn't authenticate your user", code=401) if not user.is_active: raise ProcessingException("User is deactivated", code=401) login_user_bundle(user) if not current_user.is_authenticated: - raise ProcessingException(description='Not authenticated!', code=401) + raise ProcessingException(description="Not authenticated!", code=401) + -class AbstractProcessor(): +class AbstractProcessor: """Abstract processors for the Web services. """ def is_authorized(self, user, obj): if user.id != obj.user_id: - raise ProcessingException(description='Not Authorized', code=401) + raise ProcessingException(description="Not Authorized", code=401) def get_single_preprocessor(self, instance_id=None, **kw): # Check if the user is authorized to modify the specified @@ -69,13 +69,11 @@ class AbstractProcessor(): """Accepts a single argument, `search_params`, which is a dictionary containing the search parameters for the request. """ - filt = dict(name="user_id", - op="eq", - val=current_user.id) + filt = dict(name="user_id", op="eq", val=current_user.id) # Check if there are any filters there already. if "filters" not in search_params: - search_params["filters"] = [] + search_params["filters"] = [] search_params["filters"].append(filt) @@ -95,13 +93,11 @@ class AbstractProcessor(): is a dictionary representing the fields to change on the matching instances and the values to which they will be set. """ - filt = dict(name="user_id", - op="eq", - val=current_user.id) + filt = dict(name="user_id", op="eq", val=current_user.id) # Check if there are any filters there already. if "filters" not in search_params: - search_params["filters"] = [] + search_params["filters"] = [] search_params["filters"].append(filt) diff --git a/newspipe/web/views/api/v3/feed.py b/newspipe/web/views/api/v3/feed.py index 2cbbafd9..9d2c9180 100644 --- a/newspipe/web/views/api/v3/feed.py +++ b/newspipe/web/views/api/v3/feed.py @@ -33,6 +33,7 @@ from web.controllers import FeedController from web.views.api.v3.common import AbstractProcessor from web.views.api.v3.common import url_prefix, auth_func + class FeedProcessor(AbstractProcessor): """Concrete processors for the Feed Web service. """ @@ -43,16 +44,19 @@ class FeedProcessor(AbstractProcessor): feed = FeedController(current_user.id).get(id=instance_id) self.is_authorized(current_user, feed) + feed_processor = FeedProcessor() -blueprint_feed = manager.create_api_blueprint(models.Feed, - url_prefix=url_prefix, - methods=['GET', 'POST', 'PUT', 'DELETE'], - preprocessors=dict(GET_SINGLE=[auth_func, - feed_processor.get_single_preprocessor], - GET_MANY=[auth_func, - feed_processor.get_many_preprocessor], - PUT_SINGLE=[auth_func], - POST=[auth_func], - DELETE=[auth_func])) +blueprint_feed = manager.create_api_blueprint( + models.Feed, + url_prefix=url_prefix, + methods=["GET", "POST", "PUT", "DELETE"], + preprocessors=dict( + GET_SINGLE=[auth_func, feed_processor.get_single_preprocessor], + GET_MANY=[auth_func, feed_processor.get_many_preprocessor], + PUT_SINGLE=[auth_func], + POST=[auth_func], + DELETE=[auth_func], + ), +) application.register_blueprint(blueprint_feed) diff --git a/newspipe/web/views/article.py b/newspipe/web/views/article.py index bf39795d..ba66726e 100644 --- a/newspipe/web/views/article.py +++ b/newspipe/web/views/article.py @@ -1,6 +1,14 @@ from datetime import datetime, timedelta -from flask import (Blueprint, g, render_template, redirect, - flash, url_for, make_response, request) +from flask import ( + Blueprint, + g, + render_template, + redirect, + flash, + url_for, + make_response, + request, +) from flask_babel import gettext from flask_login import login_required, current_user @@ -9,25 +17,24 @@ from flask_login import login_required, current_user from bootstrap import db from lib.utils import clear_string, redirect_url from lib.data import export_json -from web.controllers import (ArticleController, UserController, - CategoryController) +from web.controllers import ArticleController, UserController, CategoryController from web.lib.view_utils import etag_match -articles_bp = Blueprint('articles', __name__, url_prefix='/articles') -article_bp = Blueprint('article', __name__, url_prefix='/article') +articles_bp = Blueprint("articles", __name__, url_prefix="/articles") +article_bp = Blueprint("article", __name__, url_prefix="/article") -@article_bp.route('/redirect/<int:article_id>', methods=['GET']) +@article_bp.route("/redirect/<int:article_id>", methods=["GET"]) @login_required def redirect_to_article(article_id): contr = ArticleController(current_user.id) article = contr.get(id=article_id) if not article.readed: - contr.update({'id': article.id}, {'readed': True}) + contr.update({"id": article.id}, {"readed": True}) return redirect(article.link) -@article_bp.route('/<int:article_id>', methods=['GET']) +@article_bp.route("/<int:article_id>", methods=["GET"]) @login_required @etag_match def article(article_id=None): @@ -35,11 +42,12 @@ def article(article_id=None): Presents an article. """ article = ArticleController(current_user.id).get(id=article_id) - return render_template('article.html', - head_titles=[clear_string(article.title)], - article=article) + return render_template( + "article.html", head_titles=[clear_string(article.title)], article=article + ) -@article_bp.route('/public/<int:article_id>', methods=['GET']) + +@article_bp.route("/public/<int:article_id>", methods=["GET"]) @etag_match def article_pub(article_id=None): """ @@ -48,13 +56,13 @@ def article_pub(article_id=None): """ article = ArticleController().get(id=article_id) if article.source.private or not article.source.user.is_public_profile: - return render_template('errors/404.html'), 404 - return render_template('article_pub.html', - head_titles=[clear_string(article.title)], - article=article) + return render_template("errors/404.html"), 404 + return render_template( + "article_pub.html", head_titles=[clear_string(article.title)], article=article + ) -@article_bp.route('/like/<int:article_id>', methods=['GET']) +@article_bp.route("/like/<int:article_id>", methods=["GET"]) @login_required def like(article_id=None): """ @@ -62,80 +70,84 @@ def like(article_id=None): """ art_contr = ArticleController(current_user.id) article = art_contr.get(id=article_id) - art_contr = art_contr.update({'id': article_id}, - {'like': not article.like}) + art_contr = art_contr.update({"id": article_id}, {"like": not article.like}) return redirect(redirect_url()) -@article_bp.route('/delete/<int:article_id>', methods=['GET']) +@article_bp.route("/delete/<int:article_id>", methods=["GET"]) @login_required def delete(article_id=None): """ Delete an article from the database. """ article = ArticleController(current_user.id).delete(article_id) - flash(gettext('Article %(article_title)s deleted', - article_title=article.title), 'success') - return redirect(url_for('home')) + flash( + gettext("Article %(article_title)s deleted", article_title=article.title), + "success", + ) + return redirect(url_for("home")) -@articles_bp.route('/history', methods=['GET']) -@articles_bp.route('/history/<int:year>', methods=['GET']) -@articles_bp.route('/history/<int:year>/<int:month>', methods=['GET']) +@articles_bp.route("/history", methods=["GET"]) +@articles_bp.route("/history/<int:year>", methods=["GET"]) +@articles_bp.route("/history/<int:year>/<int:month>", methods=["GET"]) @login_required def history(year=None, month=None): cntr, artcles = ArticleController(current_user.id).get_history(year, month) - return render_template('history.html', articles_counter=cntr, - articles=artcles, year=year, month=month) + return render_template( + "history.html", articles_counter=cntr, articles=artcles, year=year, month=month + ) -@article_bp.route('/mark_as/<string:new_value>', methods=['GET']) -@article_bp.route('/mark_as/<string:new_value>/article/<int:article_id>', - methods=['GET']) +@article_bp.route("/mark_as/<string:new_value>", methods=["GET"]) +@article_bp.route( + "/mark_as/<string:new_value>/article/<int:article_id>", methods=["GET"] +) @login_required -def mark_as(new_value='read', feed_id=None, article_id=None): +def mark_as(new_value="read", feed_id=None, article_id=None): """ Mark all unreaded articles as read. """ - readed = new_value == 'read' + readed = new_value == "read" art_contr = ArticleController(current_user.id) - filters = {'readed': not readed} + filters = {"readed": not readed} if feed_id is not None: - filters['feed_id'] = feed_id - message = 'Feed marked as %s.' + filters["feed_id"] = feed_id + message = "Feed marked as %s." elif article_id is not None: - filters['id'] = article_id - message = 'Article marked as %s.' + filters["id"] = article_id + message = "Article marked as %s." else: - message = 'All article marked as %s.' + message = "All article marked as %s." art_contr.update(filters, {"readed": readed}) - flash(gettext(message % new_value), 'info') + flash(gettext(message % new_value), "info") if readed: return redirect(redirect_url()) - return redirect('home') + return redirect("home") -@articles_bp.route('/expire_articles', methods=['GET']) +@articles_bp.route("/expire_articles", methods=["GET"]) @login_required def expire(): """ Delete articles older than the given number of weeks. """ current_time = datetime.utcnow() - weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10))) + weeks_ago = current_time - timedelta(int(request.args.get("weeks", 10))) art_contr = ArticleController(current_user.id) - query = art_contr.read(__or__={'date__lt': weeks_ago, - 'retrieved_date__lt': weeks_ago}) + query = art_contr.read( + __or__={"date__lt": weeks_ago, "retrieved_date__lt": weeks_ago} + ) count = query.count() query.delete() db.session.commit() - flash(gettext('%(count)d articles deleted', count=count), 'info') + flash(gettext("%(count)d articles deleted", count=count), "info") return redirect(redirect_url()) -@articles_bp.route('/export', methods=['GET']) +@articles_bp.route("/export", methods=["GET"]) @login_required def export(): """ @@ -145,10 +157,9 @@ def export(): try: json_result = export_json(user) except Exception as e: - flash(gettext("Error when exporting articles."), 'danger') + flash(gettext("Error when exporting articles."), "danger") return redirect(redirect_url()) response = make_response(json_result) - response.mimetype = 'application/json' - response.headers["Content-Disposition"] \ - = 'attachment; filename=account.json' + response.mimetype = "application/json" + response.headers["Content-Disposition"] = "attachment; filename=account.json" return response diff --git a/newspipe/web/views/bookmark.py b/newspipe/web/views/bookmark.py index 21d832d2..bceda631 100644 --- a/newspipe/web/views/bookmark.py +++ b/newspipe/web/views/bookmark.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # Newspipe - A Web based news aggregator. # Copyright (C) 2010-2017 Cédric Bonhomme - https://www.cedricbonhomme.org @@ -30,8 +30,15 @@ import logging import datetime from werkzeug.exceptions import BadRequest -from flask import Blueprint, render_template, flash, \ - redirect, request, url_for, make_response +from flask import ( + Blueprint, + render_template, + flash, + redirect, + request, + url_for, + make_response, +) from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args @@ -46,93 +53,105 @@ from web.controllers import BookmarkController, BookmarkTagController from web.models import BookmarkTag logger = logging.getLogger(__name__) -bookmarks_bp = Blueprint('bookmarks', __name__, url_prefix='/bookmarks') -bookmark_bp = Blueprint('bookmark', __name__, url_prefix='/bookmark') +bookmarks_bp = Blueprint("bookmarks", __name__, url_prefix="/bookmarks") +bookmark_bp = Blueprint("bookmark", __name__, url_prefix="/bookmark") -@bookmarks_bp.route('/', defaults={'per_page': '50'}, methods=['GET']) -@bookmarks_bp.route('/<string:status>', defaults={'per_page': '50'}, - methods=['GET']) -def list_(per_page, status='all'): +@bookmarks_bp.route("/", defaults={"per_page": "50"}, methods=["GET"]) +@bookmarks_bp.route("/<string:status>", defaults={"per_page": "50"}, methods=["GET"]) +def list_(per_page, status="all"): "Lists the bookmarks." head_titles = [gettext("Bookmarks")] user_id = None filters = {} - tag = request.args.get('tag', None) + tag = request.args.get("tag", None) if tag: - filters['tags_proxy__contains'] = tag - query = request.args.get('query', None) + filters["tags_proxy__contains"] = tag + query = request.args.get("query", None) if query: - query_regex = '%' + query + '%' - filters['__or__'] = {'title__ilike': query_regex, - 'description__ilike': query_regex} + query_regex = "%" + query + "%" + filters["__or__"] = { + "title__ilike": query_regex, + "description__ilike": query_regex, + } if current_user.is_authenticated: # query for the bookmarks of the authenticated user user_id = current_user.id - if status == 'public': + if status == "public": # load public bookmarks only - filters['shared'] = True - elif status == 'private': + filters["shared"] = True + elif status == "private": # load private bookmarks only - filters['shared'] = False + filters["shared"] = False else: # no filter: load shared and public bookmarks pass - if status == 'unread': - filters['to_read'] = True + if status == "unread": + filters["to_read"] = True else: pass else: # query for the shared bookmarks (of all users) head_titles = [gettext("Recent bookmarks")] - not_created_before = datetime.datetime.today() - \ - datetime.timedelta(days=900) - filters['time__gt'] = not_created_before # only "recent" bookmarks - filters['shared'] = True + not_created_before = datetime.datetime.today() - datetime.timedelta(days=900) + filters["time__gt"] = not_created_before # only "recent" bookmarks + filters["shared"] = True - bookmarks = BookmarkController(user_id) \ - .read(**filters) \ - .order_by(desc('time')) + bookmarks = BookmarkController(user_id).read(**filters).order_by(desc("time")) - #tag_contr = BookmarkTagController(user_id) - #tag_contr.read().join(bookmarks).all() + # tag_contr = BookmarkTagController(user_id) + # tag_contr.read().join(bookmarks).all() page, per_page, offset = get_page_args() - pagination = Pagination(page=page, total=bookmarks.count(), - css_framework='bootstrap3', - search=False, record_name='bookmarks', - per_page=per_page) - - return render_template('bookmarks.html', - head_titles=head_titles, - bookmarks=bookmarks.offset(offset).limit(per_page), - pagination=pagination, - tag=tag, - query=query) - - -@bookmark_bp.route('/create', methods=['GET']) -@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['GET']) + pagination = Pagination( + page=page, + total=bookmarks.count(), + css_framework="bootstrap3", + search=False, + record_name="bookmarks", + per_page=per_page, + ) + + return render_template( + "bookmarks.html", + head_titles=head_titles, + bookmarks=bookmarks.offset(offset).limit(per_page), + pagination=pagination, + tag=tag, + query=query, + ) + + +@bookmark_bp.route("/create", methods=["GET"]) +@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["GET"]) @login_required def form(bookmark_id=None): "Form to create/edit bookmarks." action = gettext("Add a new bookmark") head_titles = [action] if bookmark_id is None: - return render_template('edit_bookmark.html', action=action, - head_titles=head_titles, form=BookmarkForm()) + return render_template( + "edit_bookmark.html", + action=action, + head_titles=head_titles, + form=BookmarkForm(), + ) bookmark = BookmarkController(current_user.id).get(id=bookmark_id) - action = gettext('Edit bookmark') + action = gettext("Edit bookmark") head_titles = [action] form = BookmarkForm(obj=bookmark) form.tags.data = ", ".join(bookmark.tags_proxy) - return render_template('edit_bookmark.html', action=action, - head_titles=head_titles, bookmark=bookmark, - form=form) + return render_template( + "edit_bookmark.html", + action=action, + head_titles=head_titles, + bookmark=bookmark, + form=form, + ) -@bookmark_bp.route('/create', methods=['POST']) -@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['POST']) +@bookmark_bp.route("/create", methods=["POST"]) +@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["POST"]) @login_required def process_form(bookmark_id=None): "Process the creation/edition of bookmarks." @@ -141,116 +160,131 @@ def process_form(bookmark_id=None): tag_contr = BookmarkTagController(current_user.id) if not form.validate(): - return render_template('edit_bookmark.html', form=form) + return render_template("edit_bookmark.html", form=form) - if form.title.data == '': + if form.title.data == "": title = form.href.data else: title = form.title.data - bookmark_attr = {'href': form.href.data, - 'description': form.description.data, - 'title': title, - 'shared': form.shared.data, - 'to_read': form.to_read.data} + bookmark_attr = { + "href": form.href.data, + "description": form.description.data, + "title": title, + "shared": form.shared.data, + "to_read": form.to_read.data, + } if bookmark_id is not None: tags = [] - for tag in form.tags.data.split(','): - new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id, - bookmark_id=bookmark_id) + for tag in form.tags.data.split(","): + new_tag = tag_contr.create( + text=tag.strip(), user_id=current_user.id, bookmark_id=bookmark_id + ) tags.append(new_tag) - bookmark_attr['tags'] = tags - bookmark_contr.update({'id': bookmark_id}, bookmark_attr) - flash(gettext('Bookmark successfully updated.'), 'success') - return redirect(url_for('bookmark.form', bookmark_id=bookmark_id)) + bookmark_attr["tags"] = tags + bookmark_contr.update({"id": bookmark_id}, bookmark_attr) + flash(gettext("Bookmark successfully updated."), "success") + return redirect(url_for("bookmark.form", bookmark_id=bookmark_id)) # Create a new bookmark new_bookmark = bookmark_contr.create(**bookmark_attr) tags = [] - for tag in form.tags.data.split(','): - new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id, - bookmark_id=new_bookmark.id) + for tag in form.tags.data.split(","): + new_tag = tag_contr.create( + text=tag.strip(), user_id=current_user.id, bookmark_id=new_bookmark.id + ) tags.append(new_tag) - bookmark_attr['tags'] = tags - bookmark_contr.update({'id': new_bookmark.id}, bookmark_attr) - flash(gettext('Bookmark successfully created.'), 'success') - return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id)) + bookmark_attr["tags"] = tags + bookmark_contr.update({"id": new_bookmark.id}, bookmark_attr) + flash(gettext("Bookmark successfully created."), "success") + return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id)) -@bookmark_bp.route('/delete/<int:bookmark_id>', methods=['GET']) +@bookmark_bp.route("/delete/<int:bookmark_id>", methods=["GET"]) @login_required def delete(bookmark_id=None): "Delete a bookmark." bookmark = BookmarkController(current_user.id).delete(bookmark_id) - flash(gettext("Bookmark %(bookmark_name)s successfully deleted.", - bookmark_name=bookmark.title), 'success') - return redirect(url_for('bookmarks.list_')) + flash( + gettext( + "Bookmark %(bookmark_name)s successfully deleted.", + bookmark_name=bookmark.title, + ), + "success", + ) + return redirect(url_for("bookmarks.list_")) -@bookmarks_bp.route('/delete', methods=['GET']) +@bookmarks_bp.route("/delete", methods=["GET"]) @login_required def delete_all(): "Delete all bookmarks." bookmark = BookmarkController(current_user.id).read().delete() db.session.commit() - flash(gettext("Bookmarks successfully deleted."), 'success') + flash(gettext("Bookmarks successfully deleted."), "success") return redirect(redirect_url()) -@bookmark_bp.route('/bookmarklet', methods=['GET', 'POST']) +@bookmark_bp.route("/bookmarklet", methods=["GET", "POST"]) @login_required def bookmarklet(): bookmark_contr = BookmarkController(current_user.id) - href = (request.args if request.method == 'GET' else request.form)\ - .get('href', None) + href = (request.args if request.method == "GET" else request.form).get("href", None) if not href: flash(gettext("Couldn't add bookmark: url missing."), "error") raise BadRequest("url is missing") - title = (request.args if request.method == 'GET' else request.form)\ - .get('title', None) + title = (request.args if request.method == "GET" else request.form).get( + "title", None + ) if not title: title = href - bookmark_exists = bookmark_contr.read(**{'href': href}).all() + bookmark_exists = bookmark_contr.read(**{"href": href}).all() if bookmark_exists: - flash(gettext("Couldn't add bookmark: bookmark already exists."), - "warning") - return redirect(url_for('bookmark.form', - bookmark_id=bookmark_exists[0].id)) + flash(gettext("Couldn't add bookmark: bookmark already exists."), "warning") + return redirect(url_for("bookmark.form", bookmark_id=bookmark_exists[0].id)) - bookmark_attr = {'href': href, - 'description': '', - 'title': title, - 'shared': True, - 'to_read': True} + bookmark_attr = { + "href": href, + "description": "", + "title": title, + "shared": True, + "to_read": True, + } new_bookmark = bookmark_contr.create(**bookmark_attr) - flash(gettext('Bookmark successfully created.'), 'success') - return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id)) + flash(gettext("Bookmark successfully created."), "success") + return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id)) -@bookmark_bp.route('/import_pinboard', methods=['POST']) +@bookmark_bp.route("/import_pinboard", methods=["POST"]) @login_required def import_pinboard(): - bookmarks = request.files.get('jsonfile', None) + bookmarks = request.files.get("jsonfile", None) if bookmarks: try: nb_bookmarks = import_pinboard_json(current_user, bookmarks.read()) - flash(gettext("%(nb_bookmarks)s bookmarks successfully imported.", - nb_bookmarks=nb_bookmarks), 'success') + flash( + gettext( + "%(nb_bookmarks)s bookmarks successfully imported.", + nb_bookmarks=nb_bookmarks, + ), + "success", + ) except Exception as e: - flash(gettext('Error when importing bookmarks.'), 'error') + flash(gettext("Error when importing bookmarks."), "error") return redirect(redirect_url()) -@bookmarks_bp.route('/export', methods=['GET']) +@bookmarks_bp.route("/export", methods=["GET"]) @login_required def export(): bookmarks = export_bookmarks(current_user) response = make_response(bookmarks) - response.mimetype = 'application/json' - response.headers["Content-Disposition"] \ - = 'attachment; filename=newspipe_bookmarks_export.json' + response.mimetype = "application/json" + response.headers[ + "Content-Disposition" + ] = "attachment; filename=newspipe_bookmarks_export.json" return response diff --git a/newspipe/web/views/category.py b/newspipe/web/views/category.py index 138561dd..1c897058 100644 --- a/newspipe/web/views/category.py +++ b/newspipe/web/views/category.py @@ -5,82 +5,105 @@ from flask_login import login_required, current_user from web.forms import CategoryForm from lib.utils import redirect_url from web.lib.view_utils import etag_match -from web.controllers import ArticleController, FeedController, \ - CategoryController +from web.controllers import ArticleController, FeedController, CategoryController -categories_bp = Blueprint('categories', __name__, url_prefix='/categories') -category_bp = Blueprint('category', __name__, url_prefix='/category') +categories_bp = Blueprint("categories", __name__, url_prefix="/categories") +category_bp = Blueprint("category", __name__, url_prefix="/category") -@categories_bp.route('/', methods=['GET']) +@categories_bp.route("/", methods=["GET"]) @login_required @etag_match def list_(): "Lists the subscribed feeds in a table." art_contr = ArticleController(current_user.id) - return render_template('categories.html', - categories=list(CategoryController(current_user.id).read().order_by('name')), - feeds_count=FeedController(current_user.id).count_by_category(), - unread_article_count=art_contr.count_by_category(readed=False), - article_count=art_contr.count_by_category()) + return render_template( + "categories.html", + categories=list(CategoryController(current_user.id).read().order_by("name")), + feeds_count=FeedController(current_user.id).count_by_category(), + unread_article_count=art_contr.count_by_category(readed=False), + article_count=art_contr.count_by_category(), + ) -@category_bp.route('/create', methods=['GET']) -@category_bp.route('/edit/<int:category_id>', methods=['GET']) +@category_bp.route("/create", methods=["GET"]) +@category_bp.route("/edit/<int:category_id>", methods=["GET"]) @login_required @etag_match def form(category_id=None): action = gettext("Add a category") head_titles = [action] if category_id is None: - return render_template('edit_category.html', action=action, - head_titles=head_titles, form=CategoryForm()) + return render_template( + "edit_category.html", + action=action, + head_titles=head_titles, + form=CategoryForm(), + ) category = CategoryController(current_user.id).get(id=category_id) - action = gettext('Edit category') + action = gettext("Edit category") head_titles = [action] if category.name: head_titles.append(category.name) - return render_template('edit_category.html', action=action, - head_titles=head_titles, category=category, - form=CategoryForm(obj=category)) + return render_template( + "edit_category.html", + action=action, + head_titles=head_titles, + category=category, + form=CategoryForm(obj=category), + ) -@category_bp.route('/delete/<int:category_id>', methods=['GET']) +@category_bp.route("/delete/<int:category_id>", methods=["GET"]) @login_required def delete(category_id=None): category = CategoryController(current_user.id).delete(category_id) - flash(gettext("Category %(category_name)s successfully deleted.", - category_name=category.name), 'success') + flash( + gettext( + "Category %(category_name)s successfully deleted.", + category_name=category.name, + ), + "success", + ) return redirect(redirect_url()) -@category_bp.route('/create', methods=['POST']) -@category_bp.route('/edit/<int:category_id>', methods=['POST']) +@category_bp.route("/create", methods=["POST"]) +@category_bp.route("/edit/<int:category_id>", methods=["POST"]) @login_required def process_form(category_id=None): form = CategoryForm() cat_contr = CategoryController(current_user.id) if not form.validate(): - return render_template('edit_category.html', form=form) + return render_template("edit_category.html", form=form) existing_cats = list(cat_contr.read(name=form.name.data)) if existing_cats and category_id is None: flash(gettext("Couldn't add category: already exists."), "warning") - return redirect(url_for('category.form', - category_id=existing_cats[0].id)) + return redirect(url_for("category.form", category_id=existing_cats[0].id)) # Edit an existing category - category_attr = {'name': form.name.data} + category_attr = {"name": form.name.data} if category_id is not None: - cat_contr.update({'id': category_id}, category_attr) - flash(gettext('Category %(cat_name)r successfully updated.', - cat_name=category_attr['name']), 'success') - return redirect(url_for('category.form', category_id=category_id)) + cat_contr.update({"id": category_id}, category_attr) + flash( + gettext( + "Category %(cat_name)r successfully updated.", + cat_name=category_attr["name"], + ), + "success", + ) + return redirect(url_for("category.form", category_id=category_id)) # Create a new category new_category = cat_contr.create(**category_attr) - flash(gettext('Category %(category_name)r successfully created.', - category_name=new_category.name), 'success') + flash( + gettext( + "Category %(category_name)r successfully created.", + category_name=new_category.name, + ), + "success", + ) - return redirect(url_for('category.form', category_id=new_category.id)) + return redirect(url_for("category.form", category_id=new_category.id)) diff --git a/newspipe/web/views/common.py b/newspipe/web/views/common.py index e422fd57..c2d8e2df 100644 --- a/newspipe/web/views/common.py +++ b/newspipe/web/views/common.py @@ -3,13 +3,18 @@ from functools import wraps from datetime import datetime from flask import current_app, Response from flask_login import login_user -from flask_principal import (Identity, Permission, RoleNeed, - session_identity_loader, identity_changed) +from flask_principal import ( + Identity, + Permission, + RoleNeed, + session_identity_loader, + identity_changed, +) from web.controllers import UserController from lib.utils import default_handler -admin_role = RoleNeed('admin') -api_role = RoleNeed('api') +admin_role = RoleNeed("admin") +api_role = RoleNeed("api") admin_permission = Permission(admin_role) api_permission = Permission(api_role) @@ -17,21 +22,23 @@ api_permission = Permission(api_role) def scoped_default_handler(): if admin_permission.can(): - role = 'admin' + role = "admin" elif api_permission.can(): - role = 'api' + role = "api" else: - role = 'user' + role = "user" @wraps(default_handler) def wrapper(obj): return default_handler(obj, role=role) + return wrapper def jsonify(func): """Will cast results of func as a result, and try to extract a status_code for the Response object""" + @wraps(func) def wrapper(*args, **kwargs): status_code = 200 @@ -40,8 +47,12 @@ def jsonify(func): return result elif isinstance(result, tuple): result, status_code = result - return Response(json.dumps(result, default=scoped_default_handler()), - mimetype='application/json', status=status_code) + return Response( + json.dumps(result, default=scoped_default_handler()), + mimetype="application/json", + status=status_code, + ) + return wrapper @@ -49,5 +60,4 @@ def login_user_bundle(user): login_user(user) identity_changed.send(current_app, identity=Identity(user.id)) session_identity_loader() - UserController(user.id).update( - {'id': user.id}, {'last_seen': datetime.utcnow()}) + UserController(user.id).update({"id": user.id}, {"last_seen": datetime.utcnow()}) diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py index b98a005a..592e3cbf 100644 --- a/newspipe/web/views/feed.py +++ b/newspipe/web/views/feed.py @@ -4,8 +4,15 @@ from datetime import datetime, timedelta from sqlalchemy import desc from werkzeug.exceptions import BadRequest -from flask import Blueprint, render_template, flash, \ - redirect, request, url_for, make_response +from flask import ( + Blueprint, + render_template, + flash, + redirect, + request, + url_for, + make_response, +) from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args @@ -15,24 +22,30 @@ from lib import misc_utils, utils from lib.feed_utils import construct_feed_from from web.lib.view_utils import etag_match from web.forms import AddFeedForm -from web.controllers import (UserController, CategoryController, - FeedController, ArticleController) +from web.controllers import ( + UserController, + CategoryController, + FeedController, + ArticleController, +) logger = logging.getLogger(__name__) -feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds') -feed_bp = Blueprint('feed', __name__, url_prefix='/feed') +feeds_bp = Blueprint("feeds", __name__, url_prefix="/feeds") +feed_bp = Blueprint("feed", __name__, url_prefix="/feed") -@feeds_bp.route('/', methods=['GET']) +@feeds_bp.route("/", methods=["GET"]) @login_required @etag_match def feeds(): "Lists the subscribed feeds in a table." art_contr = ArticleController(current_user.id) - return render_template('feeds.html', - feeds=FeedController(current_user.id).read().order_by('title'), - unread_article_count=art_contr.count_by_feed(readed=False), - article_count=art_contr.count_by_feed()) + return render_template( + "feeds.html", + feeds=FeedController(current_user.id).read().order_by("title"), + unread_article_count=art_contr.count_by_feed(readed=False), + article_count=art_contr.count_by_feed(), + ) def feed_view(feed_id=None, user_id=None): @@ -42,15 +55,19 @@ def feed_view(feed_id=None, user_id=None): if feed.category_id: category = CategoryController(user_id).get(id=feed.category_id) filters = {} - filters['feed_id'] = feed_id + filters["feed_id"] = feed_id articles = ArticleController(user_id).read_light(**filters) # Server-side pagination - page, per_page, offset = get_page_args(per_page_parameter='per_page') - pagination = Pagination(page=page, total=articles.count(), - css_framework='bootstrap3', - search=False, record_name='articles', - per_page=per_page) + page, per_page, offset = get_page_args(per_page_parameter="per_page") + pagination = Pagination( + page=page, + total=articles.count(), + css_framework="bootstrap3", + search=False, + record_name="articles", + per_page=per_page, + ) today = datetime.now() try: @@ -65,17 +82,22 @@ def feed_view(feed_id=None, user_id=None): average = 0 elapsed = today - last_article - return render_template('feed.html', - head_titles=[utils.clear_string(feed.title)], - feed=feed, category=category, - articles=articles.offset(offset).limit(per_page), - pagination=pagination, - first_post_date=first_article, - end_post_date=last_article, - average=average, delta=delta, elapsed=elapsed) - - -@feed_bp.route('/<int:feed_id>', methods=['GET']) + return render_template( + "feed.html", + head_titles=[utils.clear_string(feed.title)], + feed=feed, + category=category, + articles=articles.offset(offset).limit(per_page), + pagination=pagination, + first_post_date=first_article, + end_post_date=last_article, + average=average, + delta=delta, + elapsed=elapsed, + ) + + +@feed_bp.route("/<int:feed_id>", methods=["GET"]) @login_required @etag_match def feed(feed_id=None): @@ -83,7 +105,7 @@ def feed(feed_id=None): return feed_view(feed_id, current_user.id) -@feed_bp.route('/public/<int:feed_id>', methods=['GET']) +@feed_bp.route("/public/<int:feed_id>", methods=["GET"]) @etag_match def feed_pub(feed_id=None): """ @@ -92,90 +114,97 @@ def feed_pub(feed_id=None): """ feed = FeedController(None).get(id=feed_id) if feed.private or not feed.user.is_public_profile: - return render_template('errors/404.html'), 404 + return render_template("errors/404.html"), 404 return feed_view(feed_id, None) -@feed_bp.route('/delete/<feed_id>', methods=['GET']) +@feed_bp.route("/delete/<feed_id>", methods=["GET"]) @login_required def delete(feed_id=None): feed_contr = FeedController(current_user.id) feed = feed_contr.get(id=feed_id) feed_contr.delete(feed_id) - flash(gettext("Feed %(feed_title)s successfully deleted.", - feed_title=feed.title), 'success') - return redirect(url_for('home')) + flash( + gettext("Feed %(feed_title)s successfully deleted.", feed_title=feed.title), + "success", + ) + return redirect(url_for("home")) -@feed_bp.route('/reset_errors/<int:feed_id>', methods=['GET', 'POST']) +@feed_bp.route("/reset_errors/<int:feed_id>", methods=["GET", "POST"]) @login_required def reset_errors(feed_id): feed_contr = FeedController(current_user.id) feed = feed_contr.get(id=feed_id) - feed_contr.update({'id': feed_id}, {'error_count': 0, 'last_error': ''}) - flash(gettext('Feed %(feed_title)r successfully updated.', - feed_title=feed.title), 'success') - return redirect(request.referrer or url_for('home')) + feed_contr.update({"id": feed_id}, {"error_count": 0, "last_error": ""}) + flash( + gettext("Feed %(feed_title)r successfully updated.", feed_title=feed.title), + "success", + ) + return redirect(request.referrer or url_for("home")) -@feed_bp.route('/bookmarklet', methods=['GET', 'POST']) +@feed_bp.route("/bookmarklet", methods=["GET", "POST"]) @login_required def bookmarklet(): feed_contr = FeedController(current_user.id) - url = (request.args if request.method == 'GET' else request.form)\ - .get('url', None) + url = (request.args if request.method == "GET" else request.form).get("url", None) if not url: flash(gettext("Couldn't add feed: url missing."), "error") raise BadRequest("url is missing") - feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url})) + feed_exists = list(feed_contr.read(__or__={"link": url, "site_link": url})) if feed_exists: - flash(gettext("Couldn't add feed: feed already exists."), - "warning") - return redirect(url_for('feed.form', feed_id=feed_exists[0].id)) + flash(gettext("Couldn't add feed: feed already exists."), "warning") + return redirect(url_for("feed.form", feed_id=feed_exists[0].id)) try: feed = construct_feed_from(url) except requests.exceptions.ConnectionError: - flash(gettext("Impossible to connect to the address: {}.".format(url)), - "danger") - return redirect(url_for('home')) + flash( + gettext("Impossible to connect to the address: {}.".format(url)), "danger" + ) + return redirect(url_for("home")) except Exception: - logger.exception('something bad happened when fetching %r', url) - return redirect(url_for('home')) - if not feed.get('link'): - feed['enabled'] = False - flash(gettext("Couldn't find a feed url, you'll need to find a Atom or" - " RSS link manually and reactivate this feed"), - 'warning') + logger.exception("something bad happened when fetching %r", url) + return redirect(url_for("home")) + if not feed.get("link"): + feed["enabled"] = False + flash( + gettext( + "Couldn't find a feed url, you'll need to find a Atom or" + " RSS link manually and reactivate this feed" + ), + "warning", + ) feed = feed_contr.create(**feed) - flash(gettext('Feed was successfully created.'), 'success') + flash(gettext("Feed was successfully created."), "success") if feed.enabled and conf.CRAWLING_METHOD == "default": misc_utils.fetch(current_user.id, feed.id) - flash(gettext("Downloading articles for the new feed..."), 'info') - return redirect(url_for('feed.form', feed_id=feed.id)) + flash(gettext("Downloading articles for the new feed..."), "info") + return redirect(url_for("feed.form", feed_id=feed.id)) -@feed_bp.route('/update/<action>/<int:feed_id>', methods=['GET', 'POST']) -@feeds_bp.route('/update/<action>', methods=['GET', 'POST']) +@feed_bp.route("/update/<action>/<int:feed_id>", methods=["GET", "POST"]) +@feeds_bp.route("/update/<action>", methods=["GET", "POST"]) @login_required def update(action, feed_id=None): - readed = action == 'read' - filters = {'readed__ne': readed} + readed = action == "read" + filters = {"readed__ne": readed} - nb_days = request.args.get('nb_days', 0, type=int) + nb_days = request.args.get("nb_days", 0, type=int) if nb_days != 0: - filters['date__lt'] = datetime.now() - timedelta(days=nb_days) + filters["date__lt"] = datetime.now() - timedelta(days=nb_days) if feed_id: - filters['feed_id'] = feed_id - ArticleController(current_user.id).update(filters, {'readed': readed}) - flash(gettext('Feed successfully updated.'), 'success') - return redirect(request.referrer or url_for('home')) + filters["feed_id"] = feed_id + ArticleController(current_user.id).update(filters, {"readed": readed}) + flash(gettext("Feed successfully updated."), "success") + return redirect(request.referrer or url_for("home")) -@feed_bp.route('/create', methods=['GET']) -@feed_bp.route('/edit/<int:feed_id>', methods=['GET']) +@feed_bp.route("/create", methods=["GET"]) +@feed_bp.route("/edit/<int:feed_id>", methods=["GET"]) @login_required @etag_match def form(feed_id=None): @@ -185,22 +214,28 @@ def form(feed_id=None): if feed_id is None: form = AddFeedForm() form.set_category_choices(categories) - return render_template('edit_feed.html', action=action, - head_titles=head_titles, form=form) + return render_template( + "edit_feed.html", action=action, head_titles=head_titles, form=form + ) feed = FeedController(current_user.id).get(id=feed_id) form = AddFeedForm(obj=feed) form.set_category_choices(categories) - action = gettext('Edit feed') + action = gettext("Edit feed") head_titles = [action] if feed.title: head_titles.append(feed.title) - return render_template('edit_feed.html', action=action, - head_titles=head_titles, categories=categories, - form=form, feed=feed) - - -@feed_bp.route('/create', methods=['POST']) -@feed_bp.route('/edit/<int:feed_id>', methods=['POST']) + return render_template( + "edit_feed.html", + action=action, + head_titles=head_titles, + categories=categories, + form=form, + feed=feed, + ) + + +@feed_bp.route("/create", methods=["POST"]) +@feed_bp.route("/edit/<int:feed_id>", methods=["POST"]) @login_required def process_form(feed_id=None): form = AddFeedForm() @@ -208,58 +243,68 @@ def process_form(feed_id=None): form.set_category_choices(CategoryController(current_user.id).read()) if not form.validate(): - return render_template('edit_feed.html', form=form) + return render_template("edit_feed.html", form=form) existing_feeds = list(feed_contr.read(link=form.link.data)) if existing_feeds and feed_id is None: flash(gettext("Couldn't add feed: feed already exists."), "warning") - return redirect(url_for('feed.form', feed_id=existing_feeds[0].id)) + return redirect(url_for("feed.form", feed_id=existing_feeds[0].id)) # Edit an existing feed - feed_attr = {'title': form.title.data, 'enabled': form.enabled.data, - 'link': form.link.data, 'site_link': form.site_link.data, - 'filters': [], 'category_id': form.category_id.data, - 'private': form.private.data} - if not feed_attr['category_id'] or feed_attr['category_id'] == '0': - del feed_attr['category_id'] - - for filter_attr in ('type', 'pattern', 'action on', 'action'): - for i, value in enumerate( - request.form.getlist(filter_attr.replace(' ', '_'))): - if i >= len(feed_attr['filters']): - feed_attr['filters'].append({}) - feed_attr['filters'][i][filter_attr] = value + feed_attr = { + "title": form.title.data, + "enabled": form.enabled.data, + "link": form.link.data, + "site_link": form.site_link.data, + "filters": [], + "category_id": form.category_id.data, + "private": form.private.data, + } + if not feed_attr["category_id"] or feed_attr["category_id"] == "0": + del feed_attr["category_id"] + + for filter_attr in ("type", "pattern", "action on", "action"): + for i, value in enumerate(request.form.getlist(filter_attr.replace(" ", "_"))): + if i >= len(feed_attr["filters"]): + feed_attr["filters"].append({}) + feed_attr["filters"][i][filter_attr] = value if feed_id is not None: - feed_contr.update({'id': feed_id}, feed_attr) - flash(gettext('Feed %(feed_title)r successfully updated.', - feed_title=feed_attr['title']), 'success') - return redirect(url_for('feed.form', feed_id=feed_id)) + feed_contr.update({"id": feed_id}, feed_attr) + flash( + gettext( + "Feed %(feed_title)r successfully updated.", + feed_title=feed_attr["title"], + ), + "success", + ) + return redirect(url_for("feed.form", feed_id=feed_id)) # Create a new feed new_feed = feed_contr.create(**feed_attr) - flash(gettext('Feed %(feed_title)r successfully created.', - feed_title=new_feed.title), 'success') + flash( + gettext("Feed %(feed_title)r successfully created.", feed_title=new_feed.title), + "success", + ) if conf.CRAWLING_METHOD == "default": misc_utils.fetch(current_user.id, new_feed.id) - flash(gettext("Downloading articles for the new feed..."), 'info') + flash(gettext("Downloading articles for the new feed..."), "info") - return redirect(url_for('feed.form', feed_id=new_feed.id)) + return redirect(url_for("feed.form", feed_id=new_feed.id)) -@feeds_bp.route('/inactives', methods=['GET']) +@feeds_bp.route("/inactives", methods=["GET"]) @login_required def inactives(): """ List of inactive feeds. """ - nb_days = int(request.args.get('nb_days', 365)) + nb_days = int(request.args.get("nb_days", 365)) inactives = FeedController(current_user.id).get_inactives(nb_days) - return render_template('inactives.html', - inactives=inactives, nb_days=nb_days) + return render_template("inactives.html", inactives=inactives, nb_days=nb_days) -@feed_bp.route('/duplicates/<int:feed_id>', methods=['GET']) +@feed_bp.route("/duplicates/<int:feed_id>", methods=["GET"]) @login_required def duplicates(feed_id): """ @@ -267,40 +312,44 @@ def duplicates(feed_id): """ feed, duplicates = FeedController(current_user.id).get_duplicates(feed_id) if len(duplicates) == 0: - flash(gettext('No duplicates in the feed "{}".').format(feed.title), - 'info') - return redirect(url_for('home')) - return render_template('duplicates.html', duplicates=duplicates, feed=feed) + flash(gettext('No duplicates in the feed "{}".').format(feed.title), "info") + return redirect(url_for("home")) + return render_template("duplicates.html", duplicates=duplicates, feed=feed) -@feeds_bp.route('/export', methods=['GET']) +@feeds_bp.route("/export", methods=["GET"]) @login_required def export(): """ Export feeds to OPML. """ - include_disabled = request.args.get('includedisabled', '') == 'on' - include_private = request.args.get('includeprivate', '') == 'on' - include_exceeded_error_count = request.args. \ - get('includeexceedederrorcount', '') == 'on' + include_disabled = request.args.get("includedisabled", "") == "on" + include_private = request.args.get("includeprivate", "") == "on" + include_exceeded_error_count = ( + request.args.get("includeexceedederrorcount", "") == "on" + ) filter = {} if not include_disabled: - filter['enabled'] = True + filter["enabled"] = True if not include_private: - filter['private'] = False + filter["private"] = False if not include_exceeded_error_count: - filter['error_count__lt'] = conf.DEFAULT_MAX_ERROR + filter["error_count__lt"] = conf.DEFAULT_MAX_ERROR user = UserController(current_user.id).get(id=current_user.id) feeds = FeedController(current_user.id).read(**filter) - categories = {cat.id: cat.dump() - for cat in CategoryController(user.id).read()} - - response = make_response(render_template('opml.xml', - user=user, feeds=feeds, - categories=categories, - now=datetime.now())) - response.headers['Content-Type'] = 'application/xml' - response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml' + categories = {cat.id: cat.dump() for cat in CategoryController(user.id).read()} + + response = make_response( + render_template( + "opml.xml", + user=user, + feeds=feeds, + categories=categories, + now=datetime.now(), + ) + ) + response.headers["Content-Type"] = "application/xml" + response.headers["Content-Disposition"] = "attachment; filename=feeds.opml" return response diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py index dc7a361a..1f51c55c 100644 --- a/newspipe/web/views/home.py +++ b/newspipe/web/views/home.py @@ -2,8 +2,7 @@ import pytz import logging from datetime import datetime -from flask import current_app, render_template, \ - request, flash, url_for, redirect +from flask import current_app, render_template, request, flash, url_for, redirect from flask_login import login_required, current_user from flask_babel import gettext, get_locale from babel.dates import format_datetime, format_timedelta @@ -14,107 +13,128 @@ from lib import misc_utils from web.lib.view_utils import etag_match from web.views.common import jsonify -from web.controllers import FeedController, \ - ArticleController, CategoryController +from web.controllers import FeedController, ArticleController, CategoryController localize = pytz.utc.localize logger = logging.getLogger(__name__) -@current_app.route('/') +@current_app.route("/") @login_required @etag_match def home(): - return render_template('home.html', cdn=conf.CDN_ADDRESS) + return render_template("home.html", cdn=conf.CDN_ADDRESS) -@current_app.route('/menu') +@current_app.route("/menu") @login_required @etag_match @jsonify def get_menu(): now, locale = datetime.now(), get_locale() categories_order = [0] - categories = {0: {'name': 'No category', 'id': 0}} - for cat in CategoryController(current_user.id).read().order_by('name'): + categories = {0: {"name": "No category", "id": 0}} + for cat in CategoryController(current_user.id).read().order_by("name"): categories_order.append(cat.id) categories[cat.id] = cat unread = ArticleController(current_user.id).count_by_feed(readed=False) for cat_id in categories: - categories[cat_id]['unread'] = 0 - categories[cat_id]['feeds'] = [] + categories[cat_id]["unread"] = 0 + categories[cat_id]["feeds"] = [] feeds = {feed.id: feed for feed in FeedController(current_user.id).read()} for feed_id, feed in feeds.items(): - feed['created_rel'] = format_timedelta(feed.created_date - now, - add_direction=True, locale=locale) - feed['last_rel'] = format_timedelta(feed.last_retrieved - now, - add_direction=True, locale=locale) - feed['created_date'] = format_datetime(localize(feed.created_date), - locale=locale) - feed['last_retrieved'] = format_datetime(localize(feed.last_retrieved), - locale=locale) - feed['category_id'] = feed.category_id or 0 - feed['unread'] = unread.get(feed.id, 0) + feed["created_rel"] = format_timedelta( + feed.created_date - now, add_direction=True, locale=locale + ) + feed["last_rel"] = format_timedelta( + feed.last_retrieved - now, add_direction=True, locale=locale + ) + feed["created_date"] = format_datetime( + localize(feed.created_date), locale=locale + ) + feed["last_retrieved"] = format_datetime( + localize(feed.last_retrieved), locale=locale + ) + feed["category_id"] = feed.category_id or 0 + feed["unread"] = unread.get(feed.id, 0) if not feed.filters: - feed['filters'] = [] + feed["filters"] = [] if feed.icon_url: - feed['icon_url'] = url_for('icon.icon', url=feed.icon_url) - categories[feed['category_id']]['unread'] += feed['unread'] - categories[feed['category_id']]['feeds'].append(feed_id) - return {'feeds': feeds, 'categories': categories, - 'categories_order': categories_order, - 'crawling_method': conf.CRAWLING_METHOD, - 'max_error': conf.DEFAULT_MAX_ERROR, - 'error_threshold': conf.ERROR_THRESHOLD, - 'is_admin': current_user.is_admin, - 'all_unread_count': sum(unread.values())} + feed["icon_url"] = url_for("icon.icon", url=feed.icon_url) + categories[feed["category_id"]]["unread"] += feed["unread"] + categories[feed["category_id"]]["feeds"].append(feed_id) + return { + "feeds": feeds, + "categories": categories, + "categories_order": categories_order, + "crawling_method": conf.CRAWLING_METHOD, + "max_error": conf.DEFAULT_MAX_ERROR, + "error_threshold": conf.ERROR_THRESHOLD, + "is_admin": current_user.is_admin, + "all_unread_count": sum(unread.values()), + } def _get_filters(in_dict): filters = {} - query = in_dict.get('query') + query = in_dict.get("query") if query: - search_title = in_dict.get('search_title') == 'true' - search_content = in_dict.get('search_content') == 'true' + search_title = in_dict.get("search_title") == "true" + search_content = in_dict.get("search_content") == "true" if search_title: - filters['title__ilike'] = "%%%s%%" % query + filters["title__ilike"] = "%%%s%%" % query if search_content: - filters['content__ilike'] = "%%%s%%" % query + filters["content__ilike"] = "%%%s%%" % query if len(filters) == 0: - filters['title__ilike'] = "%%%s%%" % query + filters["title__ilike"] = "%%%s%%" % query if len(filters) > 1: filters = {"__or__": filters} - if in_dict.get('filter') == 'unread': - filters['readed'] = False - elif in_dict.get('filter') == 'liked': - filters['like'] = True - filter_type = in_dict.get('filter_type') - if filter_type in {'feed_id', 'category_id'} and in_dict.get('filter_id'): - filters[filter_type] = int(in_dict['filter_id']) or None + if in_dict.get("filter") == "unread": + filters["readed"] = False + elif in_dict.get("filter") == "liked": + filters["like"] = True + filter_type = in_dict.get("filter_type") + if filter_type in {"feed_id", "category_id"} and in_dict.get("filter_id"): + filters[filter_type] = int(in_dict["filter_id"]) or None return filters @jsonify def _articles_to_json(articles, fd_hash=None): now, locale = datetime.now(), get_locale() - fd_hash = {feed.id: {'title': feed.title, - 'icon_url': url_for('icon.icon', url=feed.icon_url) - if feed.icon_url else None} - for feed in FeedController(current_user.id).read()} - - return {'articles': [{'title': art.title, 'liked': art.like, - 'read': art.readed, 'article_id': art.id, 'selected': False, - 'feed_id': art.feed_id, 'category_id': art.category_id or 0, - 'feed_title': fd_hash[art.feed_id]['title'] if fd_hash else None, - 'icon_url': fd_hash[art.feed_id]['icon_url'] if fd_hash else None, - 'date': format_datetime(localize(art.date), locale=locale), - 'rel_date': format_timedelta(art.date - now, - threshold=1.1, add_direction=True, - locale=locale)} - for art in articles.limit(1000)]} - - -@current_app.route('/middle_panel') + fd_hash = { + feed.id: { + "title": feed.title, + "icon_url": url_for("icon.icon", url=feed.icon_url) + if feed.icon_url + else None, + } + for feed in FeedController(current_user.id).read() + } + + return { + "articles": [ + { + "title": art.title, + "liked": art.like, + "read": art.readed, + "article_id": art.id, + "selected": False, + "feed_id": art.feed_id, + "category_id": art.category_id or 0, + "feed_title": fd_hash[art.feed_id]["title"] if fd_hash else None, + "icon_url": fd_hash[art.feed_id]["icon_url"] if fd_hash else None, + "date": format_datetime(localize(art.date), locale=locale), + "rel_date": format_timedelta( + art.date - now, threshold=1.1, add_direction=True, locale=locale + ), + } + for art in articles.limit(1000) + ] + } + + +@current_app.route("/middle_panel") @login_required @etag_match def get_middle_panel(): @@ -124,8 +144,8 @@ def get_middle_panel(): return _articles_to_json(articles) -@current_app.route('/getart/<int:article_id>') -@current_app.route('/getart/<int:article_id>/<parse>') +@current_app.route("/getart/<int:article_id>") +@current_app.route("/getart/<int:article_id>/<parse>") @login_required @etag_match @jsonify @@ -134,28 +154,29 @@ def get_article(article_id, parse=False): contr = ArticleController(current_user.id) article = contr.get(id=article_id) if not article.readed: - article['readed'] = True - contr.update({'id': article_id}, {'readed': True}) - article['category_id'] = article.category_id or 0 + article["readed"] = True + contr.update({"id": article_id}, {"readed": True}) + article["category_id"] = article.category_id or 0 feed = FeedController(current_user.id).get(id=article.feed_id) - article['icon_url'] = url_for('icon.icon', url=feed.icon_url) \ - if feed.icon_url else None - article['date'] = format_datetime(localize(article.date), locale=locale) + article["icon_url"] = ( + url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None + ) + article["date"] = format_datetime(localize(article.date), locale=locale) return article -@current_app.route('/mark_all_as_read', methods=['PUT']) +@current_app.route("/mark_all_as_read", methods=["PUT"]) @login_required def mark_all_as_read(): filters = _get_filters(request.json) acontr = ArticleController(current_user.id) processed_articles = _articles_to_json(acontr.read_light(**filters)) - acontr.update(filters, {'readed': True}) + acontr.update(filters, {"readed": True}) return processed_articles -@current_app.route('/fetch', methods=['GET']) -@current_app.route('/fetch/<int:feed_id>', methods=['GET']) +@current_app.route("/fetch", methods=["GET"]) +@current_app.route("/fetch/<int:feed_id>", methods=["GET"]) @login_required def fetch(feed_id=None): """ @@ -166,6 +187,11 @@ def fetch(feed_id=None): misc_utils.fetch(current_user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: - flash(gettext("The manual retrieving of news is only available " + - "for administrator, on the Heroku platform."), "info") + flash( + gettext( + "The manual retrieving of news is only available " + + "for administrator, on the Heroku platform." + ), + "info", + ) return redirect(redirect_url()) diff --git a/newspipe/web/views/icon.py b/newspipe/web/views/icon.py index 64e54cab..e1de6402 100644 --- a/newspipe/web/views/icon.py +++ b/newspipe/web/views/icon.py @@ -3,13 +3,12 @@ from flask import Blueprint, Response, request from web.controllers import IconController from web.lib.view_utils import etag_match -icon_bp = Blueprint('icon', __name__, url_prefix='/icon') +icon_bp = Blueprint("icon", __name__, url_prefix="/icon") -@icon_bp.route('/', methods=['GET']) +@icon_bp.route("/", methods=["GET"]) @etag_match def icon(): - icon = IconController().get(url=request.args['url']) - headers = {'Cache-Control': 'max-age=86400', - 'Content-Type': icon.mimetype} + icon = IconController().get(url=request.args["url"]) + headers = {"Cache-Control": "max-age=86400", "Content-Type": icon.mimetype} return Response(base64.b64decode(icon.content), headers=headers) diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py index 0db76115..809825d3 100644 --- a/newspipe/web/views/session_mgmt.py +++ b/newspipe/web/views/session_mgmt.py @@ -4,14 +4,25 @@ import logging from datetime import datetime from werkzeug.security import generate_password_hash from werkzeug.exceptions import NotFound -from flask import (render_template, flash, session, request, - url_for, redirect, current_app) +from flask import ( + render_template, + flash, + session, + request, + url_for, + redirect, + current_app, +) from flask_babel import gettext, lazy_gettext -from flask_login import LoginManager, logout_user, \ - login_required, current_user -from flask_principal import (Principal, AnonymousIdentity, UserNeed, - identity_changed, identity_loaded, - session_identity_loader) +from flask_login import LoginManager, logout_user, login_required, current_user +from flask_principal import ( + Principal, + AnonymousIdentity, + UserNeed, + identity_changed, + identity_loaded, + session_identity_loader, +) import conf from web.views.common import admin_role, api_role, login_user_bundle @@ -24,9 +35,9 @@ Principal(current_app) login_manager = LoginManager() login_manager.init_app(current_app) -login_manager.login_view = 'login' -login_manager.login_message = lazy_gettext('Please log in to access this page.') -login_manager.login_message_category = 'info' +login_manager.login_view = "login" +login_manager.login_message = lazy_gettext("Please log in to access this page.") +login_manager.login_message_category = "info" logger = logging.getLogger(__name__) @@ -47,67 +58,77 @@ def on_identity_loaded(sender, identity): @login_manager.user_loader def load_user(user_id): - return UserController(user_id, ignore_context=True).get( - id=user_id, is_active=True) + return UserController(user_id, ignore_context=True).get(id=user_id, is_active=True) + @current_app.before_request def before_request(): if current_user.is_authenticated: UserController(current_user.id).update( - {'id': current_user.id}, {'last_seen': datetime.utcnow()}) + {"id": current_user.id}, {"last_seen": datetime.utcnow()} + ) + -@current_app.route('/login', methods=['GET', 'POST']) +@current_app.route("/login", methods=["GET", "POST"]) def login(): if current_user.is_authenticated: - return redirect(url_for('home')) + return redirect(url_for("home")) form = SigninForm() if form.validate_on_submit(): login_user_bundle(form.user) - return form.redirect('home') - return render_template('login.html', form=form) + return form.redirect("home") + return render_template("login.html", form=form) -@current_app.route('/logout') +@current_app.route("/logout") @login_required def logout(): # Remove the user information from the session logout_user() # Remove session keys set by Flask-Principal - for key in ('identity.name', 'identity.auth_type'): + for key in ("identity.name", "identity.auth_type"): session.pop(key, None) # Tell Flask-Principal the user is anonymous identity_changed.send(current_app, identity=AnonymousIdentity()) session_identity_loader() - return redirect(url_for('login')) + return redirect(url_for("login")) -@current_app.route('/signup', methods=['GET', 'POST']) +@current_app.route("/signup", methods=["GET", "POST"]) def signup(): if not conf.SELF_REGISTRATION: - flash(gettext('Self-registration is disabled.'), 'warning') - return redirect(url_for('home')) + flash(gettext("Self-registration is disabled."), "warning") + return redirect(url_for("home")) if current_user.is_authenticated: - return redirect(url_for('home')) + return redirect(url_for("home")) form = SignupForm() if form.validate_on_submit(): - user = UserController().create(nickname=form.nickname.data, - pwdhash=generate_password_hash(form.password.data)) + user = UserController().create( + nickname=form.nickname.data, + pwdhash=generate_password_hash(form.password.data), + ) # Send the confirmation email try: notifications.new_account_notification(user, form.email.data) except Exception as error: - flash(gettext('Problem while sending activation email: %(error)s', - error=error), 'danger') - return redirect(url_for('home')) - - flash(gettext('Your account has been created. ' - 'Check your mail to confirm it.'), 'success') - - return redirect(url_for('home')) - - return render_template('signup.html', form=form) + flash( + gettext( + "Problem while sending activation email: %(error)s", error=error + ), + "danger", + ) + return redirect(url_for("home")) + + flash( + gettext("Your account has been created. " "Check your mail to confirm it."), + "success", + ) + + return redirect(url_for("home")) + + return render_template("signup.html", form=form) diff --git a/newspipe/web/views/user.py b/newspipe/web/views/user.py index 24b73a60..10974947 100644 --- a/newspipe/web/views/user.py +++ b/newspipe/web/views/user.py @@ -1,8 +1,7 @@ import string import random from datetime import datetime, timedelta -from flask import (Blueprint, g, render_template, redirect, - flash, url_for, request) +from flask import Blueprint, g, render_template, redirect, flash, url_for, request from flask_babel import gettext from flask_login import login_required, current_user from flask_paginate import Pagination, get_page_args @@ -12,39 +11,47 @@ from notifications import notifications from lib import misc_utils from lib.data import import_opml, import_json from web.lib.user_utils import confirm_token -from web.controllers import (UserController, FeedController, ArticleController, - CategoryController, BookmarkController) +from web.controllers import ( + UserController, + FeedController, + ArticleController, + CategoryController, + BookmarkController, +) from web.forms import ProfileForm -users_bp = Blueprint('users', __name__, url_prefix='/users') -user_bp = Blueprint('user', __name__, url_prefix='/user') +users_bp = Blueprint("users", __name__, url_prefix="/users") +user_bp = Blueprint("user", __name__, url_prefix="/user") -@user_bp.route('/<string:nickname>', methods=['GET']) +@user_bp.route("/<string:nickname>", methods=["GET"]) def profile_public(nickname=None): """ Display the public profile of the user. """ - category_id = int(request.args.get('category_id', 0)) + category_id = int(request.args.get("category_id", 0)) user_contr = UserController() user = user_contr.get(nickname=nickname) if not user.is_public_profile: if current_user.is_authenticated and current_user.id == user.id: - flash(gettext('You must set your profile to public.'), 'info') - return redirect(url_for('user.profile')) + flash(gettext("You must set your profile to public."), "info") + return redirect(url_for("user.profile")) filters = {} - filters['private'] = False + filters["private"] = False if category_id: - filters['category_id'] = category_id + filters["category_id"] = category_id feeds = FeedController(user.id).read(**filters) - return render_template('profile_public.html', user=user, feeds=feeds, - selected_category_id=category_id) + return render_template( + "profile_public.html", user=user, feeds=feeds, selected_category_id=category_id + ) -@user_bp.route('/<string:nickname>/stream', defaults={'per_page': '25'}, methods=['GET']) +@user_bp.route( + "/<string:nickname>/stream", defaults={"per_page": "25"}, methods=["GET"] +) def user_stream(per_page, nickname=None): """ Display the stream of a user (list of articles of public feed). @@ -53,76 +60,80 @@ def user_stream(per_page, nickname=None): user = user_contr.get(nickname=nickname) if not user.is_public_profile: if current_user.is_authenticated and current_user.id == user.id: - flash(gettext('You must set your profile to public.'), 'info') - return redirect(url_for('user.profile')) + flash(gettext("You must set your profile to public."), "info") + return redirect(url_for("user.profile")) - category_id = int(request.args.get('category_id', 0)) + category_id = int(request.args.get("category_id", 0)) category = CategoryController().read(id=category_id).first() # Load the public feeds filters = {} - filters['private'] = False + filters["private"] = False if category_id: - filters['category_id'] = category_id + filters["category_id"] = category_id feeds = FeedController().read(**filters).all() # Re-initializes the filters to load the articles filters = {} - filters['feed_id__in'] = [feed.id for feed in feeds] + filters["feed_id__in"] = [feed.id for feed in feeds] if category: - filters['category_id'] = category_id + filters["category_id"] = category_id articles = ArticleController(user.id).read_light(**filters) # Server-side pagination - page, per_page, offset = get_page_args(per_page_parameter='per_page') - pagination = Pagination(page=page, total=articles.count(), - css_framework='bootstrap3', - search=False, record_name='articles', - per_page=per_page) - - return render_template('user_stream.html', user=user, - articles=articles.offset(offset).limit(per_page), - category=category, - pagination=pagination) - - -@user_bp.route('/management', methods=['GET', 'POST']) + page, per_page, offset = get_page_args(per_page_parameter="per_page") + pagination = Pagination( + page=page, + total=articles.count(), + css_framework="bootstrap3", + search=False, + record_name="articles", + per_page=per_page, + ) + + return render_template( + "user_stream.html", + user=user, + articles=articles.offset(offset).limit(per_page), + category=category, + pagination=pagination, + ) + + +@user_bp.route("/management", methods=["GET", "POST"]) @login_required def management(): """ Display the management page. """ - if request.method == 'POST': - if None != request.files.get('opmlfile', None): + if request.method == "POST": + if None != request.files.get("opmlfile", None): # Import an OPML file - data = request.files.get('opmlfile', None) + data = request.files.get("opmlfile", None) if not misc_utils.allowed_file(data.filename): - flash(gettext('File not allowed.'), 'danger') + flash(gettext("File not allowed."), "danger") else: try: nb = import_opml(current_user.nickname, data.read()) if conf.CRAWLING_METHOD == "classic": misc_utils.fetch(current_user.id, None) - flash(str(nb) + ' ' + gettext('feeds imported.'), - "success") - flash(gettext("Downloading articles..."), 'info') + flash(str(nb) + " " + gettext("feeds imported."), "success") + flash(gettext("Downloading articles..."), "info") except: - flash(gettext("Impossible to import the new feeds."), - "danger") - elif None != request.files.get('jsonfile', None): + flash(gettext("Impossible to import the new feeds."), "danger") + elif None != request.files.get("jsonfile", None): # Import an account - data = request.files.get('jsonfile', None) + data = request.files.get("jsonfile", None) if not misc_utils.allowed_file(data.filename): - flash(gettext('File not allowed.'), 'danger') + flash(gettext("File not allowed."), "danger") else: try: nb = import_json(current_user.nickname, data.read()) - flash(gettext('Account imported.'), "success") + flash(gettext("Account imported."), "success") except: - flash(gettext("Impossible to import the account."), - "danger") + flash(gettext("Impossible to import the account."), "danger") else: - flash(gettext('File not allowed.'), 'danger') + flash(gettext("File not allowed."), "danger") nb_feeds = FeedController(current_user.id).read().count() art_contr = ArticleController(current_user.id) @@ -130,14 +141,18 @@ def management(): nb_unread_articles = art_contr.read(readed=False).count() nb_categories = CategoryController(current_user.id).read().count() nb_bookmarks = BookmarkController(current_user.id).read().count() - return render_template('management.html', user=current_user, - nb_feeds=nb_feeds, nb_articles=nb_articles, - nb_unread_articles=nb_unread_articles, - nb_categories=nb_categories, - nb_bookmarks=nb_bookmarks) - - -@user_bp.route('/profile', methods=['GET', 'POST']) + return render_template( + "management.html", + user=current_user, + nb_feeds=nb_feeds, + nb_articles=nb_articles, + nb_unread_articles=nb_unread_articles, + nb_categories=nb_categories, + nb_bookmarks=nb_bookmarks, + ) + + +@user_bp.route("/profile", methods=["GET", "POST"]) @login_required def profile(): """ @@ -147,44 +162,54 @@ def profile(): user = user_contr.get(id=current_user.id) form = ProfileForm() - if request.method == 'POST': + if request.method == "POST": if form.validate(): try: - user_contr.update({'id': current_user.id}, - {'nickname': form.nickname.data, - 'password': form.password.data, - 'automatic_crawling': form.automatic_crawling.data, - 'is_public_profile': form.is_public_profile.data, - 'bio': form.bio.data, - 'webpage': form.webpage.data, - 'twitter': form.twitter.data}) + user_contr.update( + {"id": current_user.id}, + { + "nickname": form.nickname.data, + "password": form.password.data, + "automatic_crawling": form.automatic_crawling.data, + "is_public_profile": form.is_public_profile.data, + "bio": form.bio.data, + "webpage": form.webpage.data, + "twitter": form.twitter.data, + }, + ) except Exception as error: - flash(gettext('Problem while updating your profile: ' - '%(error)s', error=error), 'danger') + flash( + gettext( + "Problem while updating your profile: " "%(error)s", error=error + ), + "danger", + ) else: - flash(gettext('User %(nick)s successfully updated', - nick=user.nickname), 'success') - return redirect(url_for('user.profile')) + flash( + gettext("User %(nick)s successfully updated", nick=user.nickname), + "success", + ) + return redirect(url_for("user.profile")) else: - return render_template('profile.html', user=user, form=form) + return render_template("profile.html", user=user, form=form) - if request.method == 'GET': + if request.method == "GET": form = ProfileForm(obj=user) - return render_template('profile.html', user=user, form=form) + return render_template("profile.html", user=user, form=form) -@user_bp.route('/delete_account', methods=['GET']) +@user_bp.route("/delete_account", methods=["GET"]) @login_required def delete_account(): """ Delete the account of the user (with all its data). """ UserController(current_user.id).delete(current_user.id) - flash(gettext('Your account has been deleted.'), 'success') - return redirect(url_for('login')) + flash(gettext("Your account has been deleted."), "success") + return redirect(url_for("login")) -@user_bp.route('/confirm_account/<string:token>', methods=['GET']) +@user_bp.route("/confirm_account/<string:token>", methods=["GET"]) def confirm_account(token=None): """ Confirm the account of a user. @@ -196,8 +221,8 @@ def confirm_account(token=None): if nickname: user = user_contr.read(nickname=nickname).first() if user is not None: - user_contr.update({'id': user.id}, {'is_active': True}) - flash(gettext('Your account has been confirmed.'), 'success') + user_contr.update({"id": user.id}, {"is_active": True}) + flash(gettext("Your account has been confirmed."), "success") else: - flash(gettext('Impossible to confirm this account.'), 'danger') - return redirect(url_for('login')) + flash(gettext("Impossible to confirm this account."), "danger") + return redirect(url_for("login")) diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py index d587bd09..1fde12c7 100644 --- a/newspipe/web/views/views.py +++ b/newspipe/web/views/views.py @@ -2,8 +2,7 @@ import sys import logging import operator from datetime import datetime, timedelta -from flask import (request, render_template, flash, - url_for, redirect, current_app) +from flask import request, render_template, flash, url_for, redirect, current_app from flask_babel import gettext from sqlalchemy import desc @@ -20,26 +19,26 @@ logger = logging.getLogger(__name__) def authentication_required(error): if API_ROOT in request.url: return error - flash(gettext('Authentication required.'), 'info') - return redirect(url_for('login')) + flash(gettext("Authentication required."), "info") + return redirect(url_for("login")) @current_app.errorhandler(403) def authentication_failed(error): if API_ROOT in request.url: return error - flash(gettext('Forbidden.'), 'danger') - return redirect(url_for('login')) + flash(gettext("Forbidden."), "danger") + return redirect(url_for("login")) @current_app.errorhandler(404) def page_not_found(error): - return render_template('errors/404.html'), 404 + return render_template("errors/404.html"), 404 @current_app.errorhandler(500) def internal_server_error(error): - return render_template('errors/500.html'), 500 + return render_template("errors/500.html"), 500 @current_app.errorhandler(AssertionError) @@ -47,7 +46,7 @@ def handle_sqlalchemy_assertion_error(error): return error.args[0], 400 -@current_app.route('/popular', methods=['GET']) +@current_app.route("/popular", methods=["GET"]) @etag_match def popular(): """ @@ -57,11 +56,12 @@ def popular(): # 'not_created_before' # ie: not_added_before = date_last_added_feed - nb_days try: - nb_days = int(request.args.get('nb_days', 365)) + nb_days = int(request.args.get("nb_days", 365)) except ValueError: nb_days = 10000 - last_added_feed = FeedController().read().\ - order_by(desc('created_date')).limit(1).all() + last_added_feed = ( + FeedController().read().order_by(desc("created_date")).limit(1).all() + ) if last_added_feed: date_last_added_feed = last_added_feed[0].created_date else: @@ -69,25 +69,27 @@ def popular(): not_added_before = date_last_added_feed - timedelta(days=nb_days) filters = {} - filters['created_date__gt'] = not_added_before - filters['private'] = False - filters['error_count__lt'] = conf.DEFAULT_MAX_ERROR + filters["created_date__gt"] = not_added_before + filters["private"] = False + filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR feeds = FeedController().count_by_link(**filters) - sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), - reverse=True) - return render_template('popular.html', popular=sorted_feeds) + sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True) + return render_template("popular.html", popular=sorted_feeds) -@current_app.route('/about', methods=['GET']) +@current_app.route("/about", methods=["GET"]) @etag_match def about(): - return render_template('about.html', contact=ADMIN_EMAIL) + return render_template("about.html", contact=ADMIN_EMAIL) -@current_app.route('/about/more', methods=['GET']) + +@current_app.route("/about/more", methods=["GET"]) @etag_match def about_more(): - return render_template('about_more.html', - newspipe_version=__version__.split()[1], - registration=[conf.SELF_REGISTRATION and 'Open' or 'Closed'][0], - python_version="{}.{}.{}".format(*sys.version_info[:3]), - nb_users=UserController().read().count()) + return render_template( + "about_more.html", + newspipe_version=__version__.split()[1], + registration=[conf.SELF_REGISTRATION and "Open" or "Closed"][0], + python_version="{}.{}.{}".format(*sys.version_info[:3]), + nb_users=UserController().read().count(), + ) |