aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2020-02-26 15:20:54 +0100
committerCédric Bonhomme <cedric@cedricbonhomme.org>2020-02-26 15:20:54 +0100
commitc0d48f8a060fa30107183ad024e8c03cfee0eb26 (patch)
treee1aab8c092be0c2bd1e51d6ac09187096490652f
parentRemoved sendgrid dependency. (diff)
downloadnewspipe-c0d48f8a060fa30107183ad024e8c03cfee0eb26.tar.gz
newspipe-c0d48f8a060fa30107183ad024e8c03cfee0eb26.tar.bz2
newspipe-c0d48f8a060fa30107183ad024e8c03cfee0eb26.zip
added a little black magic
-rw-r--r--newspipe/bootstrap.py61
-rw-r--r--newspipe/conf.py123
-rw-r--r--newspipe/crawler/default_crawler.py76
-rw-r--r--newspipe/lib/article_utils.py158
-rw-r--r--newspipe/lib/data.py163
-rw-r--r--newspipe/lib/feed_utils.py115
-rwxr-xr-xnewspipe/lib/misc_utils.py73
-rw-r--r--newspipe/lib/utils.py53
-rwxr-xr-xnewspipe/manager.py46
-rw-r--r--newspipe/notifications/emails.py22
-rw-r--r--newspipe/notifications/notifications.py37
-rwxr-xr-xnewspipe/runserver.py19
-rw-r--r--newspipe/web/controllers/__init__.py12
-rw-r--r--newspipe/web/controllers/abstract.py90
-rw-r--r--newspipe/web/controllers/article.py65
-rw-r--r--newspipe/web/controllers/bookmark.py23
-rw-r--r--newspipe/web/controllers/category.py5
-rw-r--r--newspipe/web/controllers/feed.py47
-rw-r--r--newspipe/web/controllers/icon.py14
-rw-r--r--newspipe/web/controllers/user.py10
-rw-r--r--newspipe/web/decorators.py3
-rw-r--r--newspipe/web/forms.py174
-rw-r--r--newspipe/web/lib/user_utils.py12
-rw-r--r--newspipe/web/lib/view_utils.py10
-rw-r--r--newspipe/web/models/__init__.py29
-rw-r--r--newspipe/web/models/article.py48
-rw-r--r--newspipe/web/models/bookmark.py22
-rw-r--r--newspipe/web/models/category.py13
-rw-r--r--newspipe/web/models/feed.py44
-rw-r--r--newspipe/web/models/right_mixin.py27
-rw-r--r--newspipe/web/models/role.py3
-rw-r--r--newspipe/web/models/tag.py24
-rw-r--r--newspipe/web/models/user.py32
-rw-r--r--newspipe/web/views/__init__.py23
-rw-r--r--newspipe/web/views/admin.py120
-rw-r--r--newspipe/web/views/api/v2/__init__.py2
-rw-r--r--newspipe/web/views/api/v2/article.py32
-rw-r--r--newspipe/web/views/api/v2/category.py15
-rw-r--r--newspipe/web/views/api/v2/common.py64
-rw-r--r--newspipe/web/views/api/v2/feed.py34
-rw-r--r--newspipe/web/views/api/v3/__init__.py2
-rw-r--r--newspipe/web/views/api/v3/article.py36
-rw-r--r--newspipe/web/views/api/v3/common.py28
-rw-r--r--newspipe/web/views/api/v3/feed.py24
-rw-r--r--newspipe/web/views/article.py115
-rw-r--r--newspipe/web/views/bookmark.py246
-rw-r--r--newspipe/web/views/category.py91
-rw-r--r--newspipe/web/views/common.py32
-rw-r--r--newspipe/web/views/feed.py309
-rw-r--r--newspipe/web/views/home.py180
-rw-r--r--newspipe/web/views/icon.py9
-rw-r--r--newspipe/web/views/session_mgmt.py93
-rw-r--r--newspipe/web/views/user.py193
-rw-r--r--newspipe/web/views/views.py54
54 files changed, 1947 insertions, 1408 deletions
diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py
index 004bebf0..01e38fbf 100644
--- a/newspipe/bootstrap.py
+++ b/newspipe/bootstrap.py
@@ -10,16 +10,27 @@ import flask_restless
from urllib.parse import urlsplit
-def set_logging(log_path=None, log_level=logging.INFO, modules=(),
- log_format='%(asctime)s %(levelname)s %(message)s'):
+def set_logging(
+ log_path=None,
+ log_level=logging.INFO,
+ modules=(),
+ log_format="%(asctime)s %(levelname)s %(message)s",
+):
if not modules:
- modules = ('root', 'bootstrap', 'runserver',
- 'web', 'crawler.default_crawler', 'manager', 'plugins')
+ modules = (
+ "root",
+ "bootstrap",
+ "runserver",
+ "web",
+ "crawler.default_crawler",
+ "manager",
+ "plugins",
+ )
if log_path:
if not os.path.exists(os.path.dirname(log_path)):
os.makedirs(os.path.dirname(log_path))
if not os.path.exists(log_path):
- open(log_path, 'w').close()
+ open(log_path, "w").close()
handler = logging.FileHandler(log_path)
else:
handler = logging.StreamHandler()
@@ -32,39 +43,40 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(),
handler.setLevel(log_level)
logger.setLevel(log_level)
+
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
# Create Flask application
-application = Flask('web')
-if os.environ.get('Newspipe_TESTING', False) == 'true':
+application = Flask("web")
+if os.environ.get("Newspipe_TESTING", False) == "true":
application.debug = logging.DEBUG
- application.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
- application.config['TESTING'] = True
+ application.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
+ application.config["TESTING"] = True
else:
application.debug = conf.LOG_LEVEL <= logging.DEBUG
- application.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
- application.config['SQLALCHEMY_DATABASE_URI'] \
- = conf.SQLALCHEMY_DATABASE_URI
- if 'postgres' in conf.SQLALCHEMY_DATABASE_URI:
- application.config['SQLALCHEMY_POOL_SIZE'] = 15
- application.config['SQLALCHEMY_MAX_OVERFLOW'] = 0
+ application.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+ application.config["SQLALCHEMY_DATABASE_URI"] = conf.SQLALCHEMY_DATABASE_URI
+ if "postgres" in conf.SQLALCHEMY_DATABASE_URI:
+ application.config["SQLALCHEMY_POOL_SIZE"] = 15
+ application.config["SQLALCHEMY_MAX_OVERFLOW"] = 0
scheme, domain, _, _, _ = urlsplit(conf.PLATFORM_URL)
-application.config['SERVER_NAME'] = domain
-application.config['PREFERRED_URL_SCHEME'] = scheme
+application.config["SERVER_NAME"] = domain
+application.config["PREFERRED_URL_SCHEME"] = scheme
set_logging(conf.LOG_PATH, log_level=conf.LOG_LEVEL)
# Create secrey key so we can use sessions
-application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None)
-if not application.config['SECRET_KEY']:
- application.config['SECRET_KEY'] = os.urandom(12)
+application.config["SECRET_KEY"] = getattr(conf, "WEBSERVER_SECRET", None)
+if not application.config["SECRET_KEY"]:
+ application.config["SECRET_KEY"] = os.urandom(12)
-application.config['SECURITY_PASSWORD_SALT'] = getattr(conf,
- 'SECURITY_PASSWORD_SALT', None)
-if not application.config['SECURITY_PASSWORD_SALT']:
- application.config['SECURITY_PASSWORD_SALT'] = os.urandom(12)
+application.config["SECURITY_PASSWORD_SALT"] = getattr(
+ conf, "SECURITY_PASSWORD_SALT", None
+)
+if not application.config["SECURITY_PASSWORD_SALT"]:
+ application.config["SECURITY_PASSWORD_SALT"] = os.urandom(12)
db = SQLAlchemy(application)
@@ -74,5 +86,6 @@ manager = flask_restless.APIManager(application, flask_sqlalchemy_db=db)
def populate_g():
from flask import g
+
g.db = db
g.app = application
diff --git a/newspipe/conf.py b/newspipe/conf.py
index fcdaef75..bb51e563 100644
--- a/newspipe/conf.py
+++ b/newspipe/conf.py
@@ -10,41 +10,36 @@ import logging
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
PATH = os.path.abspath(".")
-API_ROOT = '/api/v2.0'
+API_ROOT = "/api/v2.0"
# available languages
-LANGUAGES = {
- 'en': 'English',
- 'fr': 'French'
-}
+LANGUAGES = {"en": "English", "fr": "French"}
-TIME_ZONE = {
- "en": "US/Eastern",
- "fr": "Europe/Paris"
-}
+TIME_ZONE = {"en": "US/Eastern", "fr": "Europe/Paris"}
-DEFAULTS = {"platform_url": "https://www.newspipe.org/",
- "self_registration": "false",
- "cdn_address": "",
- "admin_email": "info@newspipe.org",
- "token_validity_period": "3600",
- "default_max_error": "3",
- "log_path": "newspipe.log",
- "log_level": "info",
- "secret_key": "",
- "security_password_salt": "",
- "enabled": "false",
- "notification_email": "info@newspipe.org",
- "tls": "false",
- "ssl": "true",
- "host": "0.0.0.0",
- "port": "5000",
- "crawling_method": "default",
- "crawler_user_agent": "Newspipe (https://github.com/newspipe)",
- "crawler_timeout": "30",
- "crawler_resolv": "false",
- "feed_refresh_interval": "120"
- }
+DEFAULTS = {
+ "platform_url": "https://www.newspipe.org/",
+ "self_registration": "false",
+ "cdn_address": "",
+ "admin_email": "info@newspipe.org",
+ "token_validity_period": "3600",
+ "default_max_error": "3",
+ "log_path": "newspipe.log",
+ "log_level": "info",
+ "secret_key": "",
+ "security_password_salt": "",
+ "enabled": "false",
+ "notification_email": "info@newspipe.org",
+ "tls": "false",
+ "ssl": "true",
+ "host": "0.0.0.0",
+ "port": "5000",
+ "crawling_method": "default",
+ "crawler_user_agent": "Newspipe (https://github.com/newspipe)",
+ "crawler_timeout": "30",
+ "crawler_resolv": "false",
+ "feed_refresh_interval": "120",
+}
# load the configuration
@@ -52,51 +47,53 @@ config = confparser.SafeConfigParser(defaults=DEFAULTS)
config.read(os.path.join(BASE_DIR, "conf/conf.cfg"))
-WEBSERVER_HOST = config.get('webserver', 'host')
-WEBSERVER_PORT = config.getint('webserver', 'port')
-WEBSERVER_SECRET = config.get('webserver', 'secret_key')
-WEBSERVER_DEBUG = config.getboolean('webserver', 'debug')
+WEBSERVER_HOST = config.get("webserver", "host")
+WEBSERVER_PORT = config.getint("webserver", "port")
+WEBSERVER_SECRET = config.get("webserver", "secret_key")
+WEBSERVER_DEBUG = config.getboolean("webserver", "debug")
-CDN_ADDRESS = config.get('cdn', 'cdn_address')
+CDN_ADDRESS = config.get("cdn", "cdn_address")
try:
- PLATFORM_URL = config.get('misc', 'platform_url')
+ PLATFORM_URL = config.get("misc", "platform_url")
except:
PLATFORM_URL = "https://www.newspipe.org/"
-ADMIN_EMAIL = config.get('misc', 'admin_email')
-SELF_REGISTRATION = config.getboolean('misc', 'self_registration')
-SECURITY_PASSWORD_SALT = config.get('misc', 'security_password_salt')
+ADMIN_EMAIL = config.get("misc", "admin_email")
+SELF_REGISTRATION = config.getboolean("misc", "self_registration")
+SECURITY_PASSWORD_SALT = config.get("misc", "security_password_salt")
try:
- TOKEN_VALIDITY_PERIOD = config.getint('misc', 'token_validity_period')
+ TOKEN_VALIDITY_PERIOD = config.getint("misc", "token_validity_period")
except:
- TOKEN_VALIDITY_PERIOD = int(config.get('misc', 'token_validity_period'))
-LOG_PATH = os.path.abspath(config.get('misc', 'log_path'))
-LOG_LEVEL = {'debug': logging.DEBUG,
- 'info': logging.INFO,
- 'warn': logging.WARN,
- 'error': logging.ERROR,
- 'fatal': logging.FATAL}[config.get('misc', 'log_level')]
+ TOKEN_VALIDITY_PERIOD = int(config.get("misc", "token_validity_period"))
+LOG_PATH = os.path.abspath(config.get("misc", "log_path"))
+LOG_LEVEL = {
+ "debug": logging.DEBUG,
+ "info": logging.INFO,
+ "warn": logging.WARN,
+ "error": logging.ERROR,
+ "fatal": logging.FATAL,
+}[config.get("misc", "log_level")]
-SQLALCHEMY_DATABASE_URI = config.get('database', 'database_url')
+SQLALCHEMY_DATABASE_URI = config.get("database", "database_url")
-CRAWLING_METHOD = config.get('crawler', 'crawling_method')
-CRAWLER_USER_AGENT = config.get('crawler', 'user_agent')
-DEFAULT_MAX_ERROR = config.getint('crawler', 'default_max_error')
+CRAWLING_METHOD = config.get("crawler", "crawling_method")
+CRAWLER_USER_AGENT = config.get("crawler", "user_agent")
+DEFAULT_MAX_ERROR = config.getint("crawler", "default_max_error")
ERROR_THRESHOLD = int(DEFAULT_MAX_ERROR / 2)
-CRAWLER_TIMEOUT = config.get('crawler', 'timeout')
-CRAWLER_RESOLV = config.getboolean('crawler', 'resolv')
+CRAWLER_TIMEOUT = config.get("crawler", "timeout")
+CRAWLER_RESOLV = config.getboolean("crawler", "resolv")
try:
- FEED_REFRESH_INTERVAL = config.getint('crawler', 'feed_refresh_interval')
+ FEED_REFRESH_INTERVAL = config.getint("crawler", "feed_refresh_interval")
except:
- FEED_REFRESH_INTERVAL = int(config.get('crawler', 'feed_refresh_interval'))
+ FEED_REFRESH_INTERVAL = int(config.get("crawler", "feed_refresh_interval"))
-NOTIFICATION_EMAIL = config.get('notification', 'notification_email')
-NOTIFICATION_HOST = config.get('notification', 'host')
-NOTIFICATION_PORT = config.getint('notification', 'port')
-NOTIFICATION_TLS = config.getboolean('notification', 'tls')
-NOTIFICATION_SSL = config.getboolean('notification', 'ssl')
-NOTIFICATION_USERNAME = config.get('notification', 'username')
-NOTIFICATION_PASSWORD = config.get('notification', 'password')
+NOTIFICATION_EMAIL = config.get("notification", "notification_email")
+NOTIFICATION_HOST = config.get("notification", "host")
+NOTIFICATION_PORT = config.getint("notification", "port")
+NOTIFICATION_TLS = config.getboolean("notification", "tls")
+NOTIFICATION_SSL = config.getboolean("notification", "ssl")
+NOTIFICATION_USERNAME = config.get("notification", "username")
+NOTIFICATION_PASSWORD = config.get("notification", "password")
CSRF_ENABLED = True
# slow database query threshold (in seconds)
diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py
index 79a746b5..0ad9f4fc 100644
--- a/newspipe/crawler/default_crawler.py
+++ b/newspipe/crawler/default_crawler.py
@@ -40,8 +40,7 @@ from web.models import User
from web.controllers import FeedController, ArticleController
from lib.utils import jarr_get
from lib.feed_utils import construct_feed_from, is_parsing_ok
-from lib.article_utils import construct_article, extract_id, \
- get_article_content
+from lib.article_utils import construct_article, extract_id, get_article_content
logger = logging.getLogger(__name__)
@@ -57,12 +56,12 @@ async def parse_feed(user, feed):
up_feed = {}
articles = []
resp = None
- #with (await sem):
+ # with (await sem):
try:
- logger.info('Retrieving feed {}'.format(feed.link))
+ logger.info("Retrieving feed {}".format(feed.link))
resp = await jarr_get(feed.link, timeout=5)
except Exception as e:
- logger.info('Problem when reading feed {}'.format(feed.link))
+ logger.info("Problem when reading feed {}".format(feed.link))
return
finally:
if None is resp:
@@ -71,38 +70,38 @@ async def parse_feed(user, feed):
content = io.BytesIO(resp.content)
parsed_feed = feedparser.parse(content)
except Exception as e:
- up_feed['last_error'] = str(e)
- up_feed['error_count'] = feed.error_count + 1
+ up_feed["last_error"] = str(e)
+ up_feed["error_count"] = feed.error_count + 1
logger.exception("error when parsing feed: " + str(e))
finally:
- up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
+ up_feed["last_retrieved"] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
try:
- FeedController().update({'id': feed.id}, up_feed)
+ FeedController().update({"id": feed.id}, up_feed)
except Exception as e:
- logger.exception('something bad here: ' + str(e))
+ logger.exception("something bad here: " + str(e))
return
if not is_parsing_ok(parsed_feed):
- up_feed['last_error'] = str(parsed_feed['bozo_exception'])
- up_feed['error_count'] = feed.error_count + 1
- FeedController().update({'id': feed.id}, up_feed)
+ up_feed["last_error"] = str(parsed_feed["bozo_exception"])
+ up_feed["error_count"] = feed.error_count + 1
+ FeedController().update({"id": feed.id}, up_feed)
return
- if parsed_feed['entries'] != []:
- articles = parsed_feed['entries']
+ if parsed_feed["entries"] != []:
+ articles = parsed_feed["entries"]
- up_feed['error_count'] = 0
- up_feed['last_error'] = ""
+ up_feed["error_count"] = 0
+ up_feed["last_error"] = ""
# Feed information
try:
construct_feed_from(feed.link, parsed_feed).update(up_feed)
except:
- logger.exception('error when constructing feed: {}'.format(feed.link))
- if feed.title and 'title' in up_feed:
+ logger.exception("error when constructing feed: {}".format(feed.link))
+ if feed.title and "title" in up_feed:
# do not override the title set by the user
- del up_feed['title']
- FeedController().update({'id': feed.id}, up_feed)
+ del up_feed["title"]
+ FeedController().update({"id": feed.id}, up_feed)
return articles
@@ -116,19 +115,18 @@ async def insert_articles(queue, nḅ_producers=1):
if item is None:
nb_producers_done += 1
if nb_producers_done == nḅ_producers:
- print('All producers done.')
- print('Process finished.')
+ print("All producers done.")
+ print("Process finished.")
break
continue
user, feed, articles = item
-
if None is articles:
- logger.info('None')
+ logger.info("None")
articles = []
- logger.info('Inserting articles for {}'.format(feed.link))
+ logger.info("Inserting articles for {}".format(feed.link))
art_contr = ArticleController(user.id)
for article in articles:
@@ -136,9 +134,8 @@ async def insert_articles(queue, nḅ_producers=1):
try:
existing_article_req = art_contr.read(
- user_id=user.id,
- feed_id=feed.id,
- entry_id=extract_id(article))
+ user_id=user.id, feed_id=feed.id, entry_id=extract_id(article)
+ )
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
@@ -149,9 +146,9 @@ async def insert_articles(queue, nḅ_producers=1):
# insertion of the new article
try:
art_contr.create(**new_article)
- logger.info('New article added: {}'.format(new_article['link']))
+ logger.info("New article added: {}".format(new_article["link"]))
except Exception:
- logger.exception('Error when inserting article in database.')
+ logger.exception("Error when inserting article in database.")
continue
@@ -160,19 +157,20 @@ async def retrieve_feed(queue, users, feed_id=None):
Launch the processus.
"""
for user in users:
- logger.info('Starting to retrieve feeds for {}'.format(user.nickname))
+ logger.info("Starting to retrieve feeds for {}".format(user.nickname))
filters = {}
- filters['user_id'] = user.id
+ filters["user_id"] = user.id
if feed_id is not None:
- filters['id'] = feed_id
- filters['enabled'] = True
- filters['error_count__lt'] = conf.DEFAULT_MAX_ERROR
- filters['last_retrieved__lt'] = datetime.now() - \
- timedelta(minutes=conf.FEED_REFRESH_INTERVAL)
+ filters["id"] = feed_id
+ filters["enabled"] = True
+ filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR
+ filters["last_retrieved__lt"] = datetime.now() - timedelta(
+ minutes=conf.FEED_REFRESH_INTERVAL
+ )
feeds = FeedController().read(**filters).all()
if feeds == []:
- logger.info('No feed to retrieve for {}'.format(user.nickname))
+ logger.info("No feed to retrieve for {}".format(user.nickname))
for feed in feeds:
articles = await parse_feed(user, feed)
diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py
index 9891e29f..c2494c79 100644
--- a/newspipe/lib/article_utils.py
+++ b/newspipe/lib/article_utils.py
@@ -13,69 +13,77 @@ import conf
from lib.utils import jarr_get
logger = logging.getLogger(__name__)
-PROCESSED_DATE_KEYS = {'published', 'created', 'updated'}
+PROCESSED_DATE_KEYS = {"published", "created", "updated"}
def extract_id(entry):
""" extract a value from an entry that will identify it among the other of
that feed"""
- return entry.get('entry_id') or entry.get('id') or entry['link']
+ return entry.get("entry_id") or entry.get("id") or entry["link"]
async def construct_article(entry, feed, fields=None, fetch=True):
"Safe method to transform a feedparser entry into an article"
now = datetime.utcnow()
article = {}
+
def push_in_article(key, value):
if not fields or key in fields:
article[key] = value
- push_in_article('feed_id', feed.id)
- push_in_article('user_id', feed.user_id)
- push_in_article('entry_id', extract_id(entry))
- push_in_article('retrieved_date', now)
- if not fields or 'date' in fields:
+
+ push_in_article("feed_id", feed.id)
+ push_in_article("user_id", feed.user_id)
+ push_in_article("entry_id", extract_id(entry))
+ push_in_article("retrieved_date", now)
+ if not fields or "date" in fields:
for date_key in PROCESSED_DATE_KEYS:
if entry.get(date_key):
try:
- article['date'] = dateutil.parser.parse(entry[date_key])\
- .astimezone(timezone.utc)
+ article["date"] = dateutil.parser.parse(entry[date_key]).astimezone(
+ timezone.utc
+ )
except Exception as e:
logger.exception(e)
else:
break
- push_in_article('content', get_article_content(entry))
- if fields is None or {'link', 'title'}.intersection(fields):
+ push_in_article("content", get_article_content(entry))
+ if fields is None or {"link", "title"}.intersection(fields):
link, title = await get_article_details(entry, fetch)
- push_in_article('link', link)
- push_in_article('title', title)
- if 'content' in article:
- #push_in_article('content', clean_urls(article['content'], link))
- push_in_article('content', article['content'])
- push_in_article('tags', {tag.get('term').strip()
- for tag in entry.get('tags', []) \
- if tag and tag.get('term', False)})
+ push_in_article("link", link)
+ push_in_article("title", title)
+ if "content" in article:
+ # push_in_article('content', clean_urls(article['content'], link))
+ push_in_article("content", article["content"])
+ push_in_article(
+ "tags",
+ {
+ tag.get("term").strip()
+ for tag in entry.get("tags", [])
+ if tag and tag.get("term", False)
+ },
+ )
return article
def get_article_content(entry):
- content = ''
- if entry.get('content'):
- content = entry['content'][0]['value']
- elif entry.get('summary'):
- content = entry['summary']
+ content = ""
+ if entry.get("content"):
+ content = entry["content"][0]["value"]
+ elif entry.get("summary"):
+ content = entry["summary"]
return content
async def get_article_details(entry, fetch=True):
- article_link = entry.get('link')
- article_title = html.unescape(entry.get('title', ''))
+ article_link = entry.get("link")
+ article_title = html.unescape(entry.get("title", ""))
if fetch and conf.CRAWLER_RESOLV and article_link or not article_title:
try:
# resolves URL behind proxies (like feedproxy.google.com)
response = await jarr_get(article_link, timeout=5)
except MissingSchema:
split, failed = urlsplit(article_link), False
- for scheme in 'https', 'http':
+ for scheme in "https", "http":
new_link = urlunsplit(SplitResult(scheme, *split[1:]))
try:
response = await jarr_get(new_link, timeout=5)
@@ -86,39 +94,44 @@ async def get_article_details(entry, fetch=True):
article_link = new_link
break
if failed:
- return article_link, article_title or 'No title'
+ return article_link, article_title or "No title"
except Exception as error:
- logger.info("Unable to get the real URL of %s. Won't fix "
- "link or title. Error: %s", article_link, error)
- return article_link, article_title or 'No title'
+ logger.info(
+ "Unable to get the real URL of %s. Won't fix "
+ "link or title. Error: %s",
+ article_link,
+ error,
+ )
+ return article_link, article_title or "No title"
article_link = response.url
if not article_title:
- bs_parsed = BeautifulSoup(response.content, 'html.parser',
- parse_only=SoupStrainer('head'))
+ bs_parsed = BeautifulSoup(
+ response.content, "html.parser", parse_only=SoupStrainer("head")
+ )
try:
- article_title = bs_parsed.find_all('title')[0].text
+ article_title = bs_parsed.find_all("title")[0].text
except IndexError: # no title
pass
- return article_link, article_title or 'No title'
+ return article_link, article_title or "No title"
class FiltersAction(Enum):
- READ = 'mark as read'
- LIKED = 'mark as favorite'
- SKIP = 'skipped'
+ READ = "mark as read"
+ LIKED = "mark as favorite"
+ SKIP = "skipped"
class FiltersType(Enum):
- REGEX = 'regex'
- MATCH = 'simple match'
- EXACT_MATCH = 'exact match'
- TAG_MATCH = 'tag match'
- TAG_CONTAINS = 'tag contains'
+ REGEX = "regex"
+ MATCH = "simple match"
+ EXACT_MATCH = "exact match"
+ TAG_MATCH = "tag match"
+ TAG_CONTAINS = "tag contains"
class FiltersTrigger(Enum):
- MATCH = 'match'
- NO_MATCH = 'no match'
+ MATCH = "match"
+ NO_MATCH = "no match"
def process_filters(filters, article, only_actions=None):
@@ -129,25 +142,30 @@ def process_filters(filters, article, only_actions=None):
for filter_ in filters:
match = False
try:
- pattern = filter_.get('pattern', '')
- filter_type = FiltersType(filter_.get('type'))
- filter_action = FiltersAction(filter_.get('action'))
- filter_trigger = FiltersTrigger(filter_.get('action on'))
+ pattern = filter_.get("pattern", "")
+ filter_type = FiltersType(filter_.get("type"))
+ filter_action = FiltersAction(filter_.get("action"))
+ filter_trigger = FiltersTrigger(filter_.get("action on"))
if filter_type is not FiltersType.REGEX:
pattern = pattern.lower()
except ValueError:
continue
if filter_action not in only_actions:
- logger.debug('ignoring filter %r' % filter_)
+ logger.debug("ignoring filter %r" % filter_)
continue
- if filter_action in {FiltersType.REGEX, FiltersType.MATCH,
- FiltersType.EXACT_MATCH} and 'title' not in article:
+ if (
+ filter_action
+ in {FiltersType.REGEX, FiltersType.MATCH, FiltersType.EXACT_MATCH}
+ and "title" not in article
+ ):
continue
- if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \
- and 'tags' not in article:
+ if (
+ filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS}
+ and "tags" not in article
+ ):
continue
- title = article.get('title', '').lower()
- tags = [tag.lower() for tag in article.get('tags', [])]
+ title = article.get("title", "").lower()
+ tags = [tag.lower() for tag in article.get("tags", [])]
if filter_type is FiltersType.REGEX:
match = re.match(pattern, title)
elif filter_type is FiltersType.MATCH:
@@ -158,8 +176,12 @@ def process_filters(filters, article, only_actions=None):
match = pattern in tags
elif filter_type is FiltersType.TAG_CONTAINS:
match = any(pattern in tag for tag in tags)
- take_action = match and filter_trigger is FiltersTrigger.MATCH \
- or not match and filter_trigger is FiltersTrigger.NO_MATCH
+ take_action = (
+ match
+ and filter_trigger is FiltersTrigger.MATCH
+ or not match
+ and filter_trigger is FiltersTrigger.NO_MATCH
+ )
if not take_action:
continue
@@ -172,15 +194,21 @@ def process_filters(filters, article, only_actions=None):
skipped = True
if skipped or read or liked:
- logger.info("%r applied on %r", filter_action.value,
- article.get('link') or article.get('title'))
+ logger.info(
+ "%r applied on %r",
+ filter_action.value,
+ article.get("link") or article.get("title"),
+ )
return skipped, read, liked
def get_skip_and_ids(entry, feed):
- entry_ids = construct_article(entry, feed,
- {'entry_id', 'feed_id', 'user_id'}, fetch=False)
- skipped, _, _ = process_filters(feed.filters,
- construct_article(entry, feed, {'title', 'tags'}, fetch=False),
- {FiltersAction.SKIP})
+ entry_ids = construct_article(
+ entry, feed, {"entry_id", "feed_id", "user_id"}, fetch=False
+ )
+ skipped, _, _ = process_filters(
+ feed.filters,
+ construct_article(entry, feed, {"title", "tags"}, fetch=False),
+ {FiltersAction.SKIP},
+ )
return skipped, entry_ids
diff --git a/newspipe/lib/data.py b/newspipe/lib/data.py
index 067a0a04..13843746 100644
--- a/newspipe/lib/data.py
+++ b/newspipe/lib/data.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2018 Cédric Bonhomme - https://www.cedricbonhomme.org
@@ -72,18 +72,28 @@ def import_opml(nickname, opml_content):
link = subscription.xmlUrl
except:
continue
- if None != Feed.query.filter(Feed.user_id == user.id, Feed.link == link).first():
+ if (
+ None
+ != Feed.query.filter(
+ Feed.user_id == user.id, Feed.link == link
+ ).first()
+ ):
continue
try:
site_link = subscription.htmlUrl
except:
site_link = ""
- new_feed = Feed(title=title, description=description,
- link=link, site_link=site_link,
- enabled=True)
+ new_feed = Feed(
+ title=title,
+ description=description,
+ link=link,
+ site_link=site_link,
+ enabled=True,
+ )
user.feeds.append(new_feed)
nb += 1
return nb
+
nb = read(subscriptions)
db.session.commit()
return nb
@@ -98,40 +108,53 @@ def import_json(nickname, json_content):
nb_feeds, nb_articles = 0, 0
# Create feeds:
for feed in json_account:
- if None != Feed.query.filter(Feed.user_id == user.id,
- Feed.link == feed["link"]).first():
+ if (
+ None
+ != Feed.query.filter(
+ Feed.user_id == user.id, Feed.link == feed["link"]
+ ).first()
+ ):
continue
- new_feed = Feed(title=feed["title"],
- description="",
- link=feed["link"],
- site_link=feed["site_link"],
- created_date=datetime.datetime.
- fromtimestamp(int(feed["created_date"])),
- enabled=feed["enabled"])
+ new_feed = Feed(
+ title=feed["title"],
+ description="",
+ link=feed["link"],
+ site_link=feed["site_link"],
+ created_date=datetime.datetime.fromtimestamp(int(feed["created_date"])),
+ enabled=feed["enabled"],
+ )
user.feeds.append(new_feed)
nb_feeds += 1
db.session.commit()
# Create articles:
for feed in json_account:
- user_feed = Feed.query.filter(Feed.user_id == user.id,
- Feed.link == feed["link"]).first()
+ user_feed = Feed.query.filter(
+ Feed.user_id == user.id, Feed.link == feed["link"]
+ ).first()
if None != user_feed:
for article in feed["articles"]:
- if None == Article.query.filter(Article.user_id == user.id,
- Article.feed_id == user_feed.id,
- Article.link == article["link"]).first():
- new_article = Article(entry_id=article["link"],
- link=article["link"],
- title=article["title"],
- content=article["content"],
- readed=article["readed"],
- like=article["like"],
- retrieved_date=datetime.datetime.
- fromtimestamp(int(article["retrieved_date"])),
- date=datetime.datetime.
- fromtimestamp(int(article["date"])),
- user_id=user.id,
- feed_id=user_feed.id)
+ if (
+ None
+ == Article.query.filter(
+ Article.user_id == user.id,
+ Article.feed_id == user_feed.id,
+ Article.link == article["link"],
+ ).first()
+ ):
+ new_article = Article(
+ entry_id=article["link"],
+ link=article["link"],
+ title=article["title"],
+ content=article["content"],
+ readed=article["readed"],
+ like=article["like"],
+ retrieved_date=datetime.datetime.fromtimestamp(
+ int(article["retrieved_date"])
+ ),
+ date=datetime.datetime.fromtimestamp(int(article["date"])),
+ user_id=user.id,
+ feed_id=user_feed.id,
+ )
user_feed.articles.append(new_article)
nb_articles += 1
db.session.commit()
@@ -144,23 +167,28 @@ def export_json(user):
"""
articles = []
for feed in user.feeds:
- articles.append({
- "title": feed.title,
- "description": feed.description,
- "link": feed.link,
- "site_link": feed.site_link,
- "enabled": feed.enabled,
- "created_date": feed.created_date.strftime('%s'),
- "articles": [ {
- "title": article.title,
- "link": article.link,
- "content": article.content,
- "readed": article.readed,
- "like": article.like,
- "date": article.date.strftime('%s'),
- "retrieved_date": article.retrieved_date.strftime('%s')
- } for article in feed.articles]
- })
+ articles.append(
+ {
+ "title": feed.title,
+ "description": feed.description,
+ "link": feed.link,
+ "site_link": feed.site_link,
+ "enabled": feed.enabled,
+ "created_date": feed.created_date.strftime("%s"),
+ "articles": [
+ {
+ "title": article.title,
+ "link": article.link,
+ "content": article.content,
+ "readed": article.readed,
+ "like": article.like,
+ "date": article.date.strftime("%s"),
+ "retrieved_date": article.retrieved_date.strftime("%s"),
+ }
+ for article in feed.articles
+ ],
+ }
+ )
return jsonify(articles)
@@ -173,19 +201,18 @@ def import_pinboard_json(user, json_content):
nb_bookmarks = 0
for bookmark in bookmarks:
tags = []
- for tag in bookmark['tags'].split(' '):
+ for tag in bookmark["tags"].split(" "):
new_tag = BookmarkTag(text=tag.strip(), user_id=user.id)
tags.append(new_tag)
bookmark_attr = {
- 'href': bookmark['href'],
- 'description': bookmark['extended'],
- 'title': bookmark['description'],
- 'shared': [bookmark['shared']=='yes' and True or False][0],
- 'to_read': [bookmark['toread']=='yes' and True or False][0],
- 'time': datetime.datetime.strptime(bookmark['time'],
- '%Y-%m-%dT%H:%M:%SZ'),
- 'tags': tags
- }
+ "href": bookmark["href"],
+ "description": bookmark["extended"],
+ "title": bookmark["description"],
+ "shared": [bookmark["shared"] == "yes" and True or False][0],
+ "to_read": [bookmark["toread"] == "yes" and True or False][0],
+ "time": datetime.datetime.strptime(bookmark["time"], "%Y-%m-%dT%H:%M:%SZ"),
+ "tags": tags,
+ }
new_bookmark = bookmark_contr.create(**bookmark_attr)
nb_bookmarks += 1
return nb_bookmarks
@@ -198,13 +225,15 @@ def export_bookmarks(user):
bookmarks = bookmark_contr.read()
export = []
for bookmark in bookmarks:
- export.append({
- 'href': bookmark.href,
- 'description': bookmark.description,
- 'title': bookmark.title,
- 'shared': 'yes' if bookmark.shared else 'no',
- 'toread': 'yes' if bookmark.to_read else 'no',
- 'time': bookmark.time.isoformat(),
- 'tags': ' '.join(bookmark.tags_proxy)
- })
+ export.append(
+ {
+ "href": bookmark.href,
+ "description": bookmark.description,
+ "title": bookmark.title,
+ "shared": "yes" if bookmark.shared else "no",
+ "toread": "yes" if bookmark.to_read else "no",
+ "time": bookmark.time.isoformat(),
+ "tags": " ".join(bookmark.tags_proxy),
+ }
+ )
return jsonify(export)
diff --git a/newspipe/lib/feed_utils.py b/newspipe/lib/feed_utils.py
index c2d4ca6e..9f1e2354 100644
--- a/newspipe/lib/feed_utils.py
+++ b/newspipe/lib/feed_utils.py
@@ -10,12 +10,17 @@ from lib.utils import try_keys, try_get_icon_url, rebuild_url
logger = logging.getLogger(__name__)
logging.captureWarnings(True)
-ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml',
- 'application/atom+xml', 'application/xml', 'text/xml')
+ACCEPTED_MIMETYPES = (
+ "application/rss+xml",
+ "application/rdf+xml",
+ "application/atom+xml",
+ "application/xml",
+ "text/xml",
+)
def is_parsing_ok(parsed_feed):
- return parsed_feed['entries'] or not parsed_feed['bozo']
+ return parsed_feed["entries"] or not parsed_feed["bozo"]
def escape_keys(*keys):
@@ -24,66 +29,71 @@ def escape_keys(*keys):
result = func(*args, **kwargs)
for key in keys:
if key in result:
- result[key] = html.unescape(result[key] or '')
+ result[key] = html.unescape(result[key] or "")
return result
+
return metawrapper
+
return wrapper
-@escape_keys('title', 'description')
+@escape_keys("title", "description")
def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
- requests_kwargs = {'headers': {'User-Agent': CRAWLER_USER_AGENT},
- 'verify': False}
+ requests_kwargs = {"headers": {"User-Agent": CRAWLER_USER_AGENT}, "verify": False}
if url is None and fp_parsed is not None:
- url = fp_parsed.get('url')
+ url = fp_parsed.get("url")
if url is not None and fp_parsed is None:
try:
response = requests.get(url, **requests_kwargs)
- fp_parsed = feedparser.parse(response.content,
- request_headers=response.headers)
+ fp_parsed = feedparser.parse(
+ response.content, request_headers=response.headers
+ )
except Exception:
- logger.exception('failed to retrieve that url')
- fp_parsed = {'bozo': True}
+ logger.exception("failed to retrieve that url")
+ fp_parsed = {"bozo": True}
assert url is not None and fp_parsed is not None
feed = feed or {}
feed_split = urllib.parse.urlsplit(url)
site_split = None
if is_parsing_ok(fp_parsed):
- feed['link'] = url
- feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link')
- feed['title'] = fp_parsed['feed'].get('title')
- feed['description'] = try_keys(fp_parsed['feed'], 'subtitle', 'title')
- feed['icon_url'] = try_keys(fp_parsed['feed'], 'icon')
+ feed["link"] = url
+ feed["site_link"] = try_keys(fp_parsed["feed"], "href", "link")
+ feed["title"] = fp_parsed["feed"].get("title")
+ feed["description"] = try_keys(fp_parsed["feed"], "subtitle", "title")
+ feed["icon_url"] = try_keys(fp_parsed["feed"], "icon")
else:
- feed['site_link'] = url
-
- if feed.get('site_link'):
- feed['site_link'] = rebuild_url(feed['site_link'], feed_split)
- site_split = urllib.parse.urlsplit(feed['site_link'])
-
- if feed.get('icon_url'):
- feed['icon_url'] = try_get_icon_url(
- feed['icon_url'], site_split, feed_split)
- if feed['icon_url'] is None:
- del feed['icon_url']
-
- if not feed.get('site_link') or not query_site \
- or all(bool(feed.get(k)) for k in ('link', 'title', 'icon_url')):
+ feed["site_link"] = url
+
+ if feed.get("site_link"):
+ feed["site_link"] = rebuild_url(feed["site_link"], feed_split)
+ site_split = urllib.parse.urlsplit(feed["site_link"])
+
+ if feed.get("icon_url"):
+ feed["icon_url"] = try_get_icon_url(feed["icon_url"], site_split, feed_split)
+ if feed["icon_url"] is None:
+ del feed["icon_url"]
+
+ if (
+ not feed.get("site_link")
+ or not query_site
+ or all(bool(feed.get(k)) for k in ("link", "title", "icon_url"))
+ ):
return feed
try:
- response = requests.get(feed['site_link'], **requests_kwargs)
+ response = requests.get(feed["site_link"], **requests_kwargs)
except requests.exceptions.InvalidSchema as e:
return feed
except:
- logger.exception('failed to retrieve %r', feed['site_link'])
+ logger.exception("failed to retrieve %r", feed["site_link"])
return feed
- bs_parsed = BeautifulSoup(response.content, 'html.parser',
- parse_only=SoupStrainer('head'))
+ bs_parsed = BeautifulSoup(
+ response.content, "html.parser", parse_only=SoupStrainer("head")
+ )
- if not feed.get('title'):
+ if not feed.get("title"):
try:
- feed['title'] = bs_parsed.find_all('title')[0].text
+ feed["title"] = bs_parsed.find_all("title")[0].text
except Exception:
pass
@@ -95,31 +105,30 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
if not all(val in elem.attrs[key] for val in vals):
return False
return True
+
return wrapper
- if not feed.get('icon_url'):
- icons = bs_parsed.find_all(check_keys(rel=['icon', 'shortcut']))
+ if not feed.get("icon_url"):
+ icons = bs_parsed.find_all(check_keys(rel=["icon", "shortcut"]))
if not len(icons):
- icons = bs_parsed.find_all(check_keys(rel=['icon']))
+ icons = bs_parsed.find_all(check_keys(rel=["icon"]))
if len(icons) >= 1:
for icon in icons:
- feed['icon_url'] = try_get_icon_url(icon.attrs['href'],
- site_split, feed_split)
- if feed['icon_url'] is not None:
+ feed["icon_url"] = try_get_icon_url(
+ icon.attrs["href"], site_split, feed_split
+ )
+ if feed["icon_url"] is not None:
break
- if feed.get('icon_url') is None:
- feed['icon_url'] = try_get_icon_url('/favicon.ico',
- site_split, feed_split)
- if 'icon_url' in feed and feed['icon_url'] is None:
- del feed['icon_url']
+ if feed.get("icon_url") is None:
+ feed["icon_url"] = try_get_icon_url("/favicon.ico", site_split, feed_split)
+ if "icon_url" in feed and feed["icon_url"] is None:
+ del feed["icon_url"]
- if not feed.get('link'):
+ if not feed.get("link"):
for type_ in ACCEPTED_MIMETYPES:
- alternates = bs_parsed.find_all(check_keys(
- rel=['alternate'], type=[type_]))
+ alternates = bs_parsed.find_all(check_keys(rel=["alternate"], type=[type_]))
if len(alternates) >= 1:
- feed['link'] = rebuild_url(alternates[0].attrs['href'],
- feed_split)
+ feed["link"] = rebuild_url(alternates[0].attrs["href"], feed_split)
break
return feed
diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py
index 8fb2d284..6fd590ac 100755
--- a/newspipe/lib/misc_utils.py
+++ b/newspipe/lib/misc_utils.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2018 Cédric Bonhomme - https://www.cedricbonhomme.org
@@ -36,6 +36,7 @@ import operator
import urllib
import subprocess
import sqlalchemy
+
try:
from urlparse import urlparse, parse_qs, urlunparse
except:
@@ -50,7 +51,7 @@ from lib.utils import clear_string
logger = logging.getLogger(__name__)
-ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json'])
+ALLOWED_EXTENSIONS = set(["xml", "opml", "json"])
def is_safe_url(target):
@@ -59,15 +60,14 @@ def is_safe_url(target):
"""
ref_url = urlparse(request.host_url)
test_url = urlparse(urljoin(request.host_url, target))
- return test_url.scheme in ('http', 'https') and \
- ref_url.netloc == test_url.netloc
+ return test_url.scheme in ("http", "https") and ref_url.netloc == test_url.netloc
def get_redirect_target():
"""
Looks at various hints to find the redirect target.
"""
- for target in request.args.get('next'), request.referrer:
+ for target in request.args.get("next"), request.referrer:
if not target:
continue
if is_safe_url(target):
@@ -78,8 +78,7 @@ def allowed_file(filename):
"""
Check if the uploaded file is allowed.
"""
- return '.' in filename and \
- filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
+ return "." in filename and filename.rsplit(".", 1)[1] in ALLOWED_EXTENSIONS
@contextmanager
@@ -100,10 +99,14 @@ def fetch(id, feed_id=None):
Fetch the feeds in a new processus.
The default crawler ("asyncio") is launched with the manager.
"""
- cmd = [sys.executable, conf.BASE_DIR + '/manager.py', 'fetch_asyncio',
- '--user_id='+str(id)]
+ cmd = [
+ sys.executable,
+ conf.BASE_DIR + "/manager.py",
+ "fetch_asyncio",
+ "--user_id=" + str(id),
+ ]
if feed_id:
- cmd.append('--feed_id='+str(feed_id))
+ cmd.append("--feed_id=" + str(feed_id))
return subprocess.Popen(cmd, stdout=subprocess.PIPE)
@@ -114,9 +117,11 @@ def history(user_id, year=None, month=None):
articles_counter = Counter()
articles = ArticleController(user_id).read()
if None != year:
- articles = articles.filter(sqlalchemy.extract('year', 'Article.date') == year)
+ articles = articles.filter(sqlalchemy.extract("year", "Article.date") == year)
if None != month:
- articles = articles.filter(sqlalchemy.extract('month', 'Article.date') == month)
+ articles = articles.filter(
+ sqlalchemy.extract("month", "Article.date") == month
+ )
for article in articles.all():
if None != year:
articles_counter[article.date.month] += 1
@@ -131,24 +136,26 @@ def clean_url(url):
"""
parsed_url = urlparse(url)
qd = parse_qs(parsed_url.query, keep_blank_values=True)
- filtered = dict((k, v) for k, v in qd.items()
- if not k.startswith('utm_'))
- return urlunparse([
- parsed_url.scheme,
- parsed_url.netloc,
- urllib.parse.quote(urllib.parse.unquote(parsed_url.path)),
- parsed_url.params,
- urllib.parse.urlencode(filtered, doseq=True),
- parsed_url.fragment
- ]).rstrip('=')
+ filtered = dict((k, v) for k, v in qd.items() if not k.startswith("utm_"))
+ return urlunparse(
+ [
+ parsed_url.scheme,
+ parsed_url.netloc,
+ urllib.parse.quote(urllib.parse.unquote(parsed_url.path)),
+ parsed_url.params,
+ urllib.parse.urlencode(filtered, doseq=True),
+ parsed_url.fragment,
+ ]
+ ).rstrip("=")
def load_stop_words():
"""
Load the stop words and return them in a list.
"""
- stop_words_lists = glob.glob(os.path.join(conf.BASE_DIR,
- 'web/var/stop_words/*.txt'))
+ stop_words_lists = glob.glob(
+ os.path.join(conf.BASE_DIR, "web/var/stop_words/*.txt")
+ )
stop_words = []
for stop_wods_list in stop_words_lists:
@@ -166,11 +173,13 @@ def top_words(articles, n=10, size=5):
"""
stop_words = load_stop_words()
words = Counter()
- wordre = re.compile(r'\b\w{%s,}\b' % size, re.I)
+ wordre = re.compile(r"\b\w{%s,}\b" % size, re.I)
for article in articles:
- for word in [elem.lower() for elem in
- wordre.findall(clear_string(article.content)) \
- if elem.lower() not in stop_words]:
+ for word in [
+ elem.lower()
+ for elem in wordre.findall(clear_string(article.content))
+ if elem.lower() not in stop_words
+ ]:
words[word] += 1
return words.most_common(n)
@@ -181,5 +190,9 @@ def tag_cloud(tags):
"""
tags.sort(key=operator.itemgetter(0))
max_tag = max([tag[1] for tag in tags])
- return '\n'.join([('<font size=%d>%s</font>' % \
- (min(1 + count * 7 / max_tag, 7), word)) for (word, count) in tags])
+ return "\n".join(
+ [
+ ("<font size=%d>%s</font>" % (min(1 + count * 7 / max_tag, 7), word))
+ for (word, count) in tags
+ ]
+ )
diff --git a/newspipe/lib/utils.py b/newspipe/lib/utils.py
index d206b769..f7244e17 100644
--- a/newspipe/lib/utils.py
+++ b/newspipe/lib/utils.py
@@ -11,18 +11,20 @@ import conf
logger = logging.getLogger(__name__)
-def default_handler(obj, role='admin'):
+def default_handler(obj, role="admin"):
"""JSON handler for default query formatting"""
- if hasattr(obj, 'isoformat'):
+ if hasattr(obj, "isoformat"):
return obj.isoformat()
- if hasattr(obj, 'dump'):
+ if hasattr(obj, "dump"):
return obj.dump(role=role)
if isinstance(obj, (set, frozenset, types.GeneratorType)):
return list(obj)
if isinstance(obj, BaseException):
return str(obj)
- raise TypeError("Object of type %s with value of %r "
- "is not JSON serializable" % (type(obj), obj))
+ raise TypeError(
+ "Object of type %s with value of %r "
+ "is not JSON serializable" % (type(obj), obj)
+ )
def try_keys(dico, *keys):
@@ -37,9 +39,12 @@ def rebuild_url(url, base_split):
if split.scheme and split.netloc:
return url # url is fine
new_split = urllib.parse.SplitResult(
- scheme=split.scheme or base_split.scheme,
- netloc=split.netloc or base_split.netloc,
- path=split.path, query='', fragment='')
+ scheme=split.scheme or base_split.scheme,
+ netloc=split.netloc or base_split.netloc,
+ path=split.path,
+ query="",
+ fragment="",
+ )
return urllib.parse.urlunsplit(new_split)
@@ -52,19 +57,22 @@ def try_get_icon_url(url, *splits):
# if html in content-type, we assume it's a fancy 404 page
try:
response = jarr_get(rb_url)
- content_type = response.headers.get('content-type', '')
+ content_type = response.headers.get("content-type", "")
except Exception:
pass
else:
- if response is not None and response.ok \
- and 'html' not in content_type and response.content:
+ if (
+ response is not None
+ and response.ok
+ and "html" not in content_type
+ and response.content
+ ):
return response.url
return None
def to_hash(text):
- return md5(text.encode('utf8') if hasattr(text, 'encode') else text)\
- .hexdigest()
+ return md5(text.encode("utf8") if hasattr(text, "encode") else text).hexdigest()
def clear_string(data):
@@ -72,18 +80,21 @@ def clear_string(data):
Clear a string by removing HTML tags, HTML special caracters
and consecutive white spaces (more that one).
"""
- p = re.compile('<[^>]+>') # HTML tags
- q = re.compile('\s') # consecutive white spaces
- return p.sub('', q.sub(' ', data))
+ p = re.compile("<[^>]+>") # HTML tags
+ q = re.compile("\s") # consecutive white spaces
+ return p.sub("", q.sub(" ", data))
-def redirect_url(default='home'):
- return request.args.get('next') or request.referrer or url_for(default)
+def redirect_url(default="home"):
+ return request.args.get("next") or request.referrer or url_for(default)
async def jarr_get(url, **kwargs):
- request_kwargs = {'verify': False, 'allow_redirects': True,
- 'timeout': conf.CRAWLER_TIMEOUT,
- 'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}}
+ request_kwargs = {
+ "verify": False,
+ "allow_redirects": True,
+ "timeout": conf.CRAWLER_TIMEOUT,
+ "headers": {"User-Agent": conf.CRAWLER_USER_AGENT},
+ }
request_kwargs.update(kwargs)
return requests.get(url, **request_kwargs)
diff --git a/newspipe/manager.py b/newspipe/manager.py
index 9535ac59..60f4c729 100755
--- a/newspipe/manager.py
+++ b/newspipe/manager.py
@@ -12,12 +12,12 @@ from flask_migrate import Migrate, MigrateCommand
import web.models
from web.controllers import UserController
-logger = logging.getLogger('manager')
+logger = logging.getLogger("manager")
Migrate(application, db)
manager = Manager(application)
-manager.add_command('db', MigrateCommand)
+manager.add_command("db", MigrateCommand)
@manager.command
@@ -30,23 +30,32 @@ def db_empty():
@manager.command
def db_create():
"Will create the database from conf parameters."
- admin = {'is_admin': True, 'is_api': True, 'is_active': True,
- 'nickname': 'admin',
- 'pwdhash': generate_password_hash(
- os.environ.get("ADMIN_PASSWORD", "password"))}
+ admin = {
+ "is_admin": True,
+ "is_api": True,
+ "is_active": True,
+ "nickname": "admin",
+ "pwdhash": generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")),
+ }
with application.app_context():
db.create_all()
UserController(ignore_context=True).create(**admin)
+
@manager.command
def create_admin(nickname, password):
"Will create an admin user."
- admin = {'is_admin': True, 'is_api': True, 'is_active': True,
- 'nickname': nickname,
- 'pwdhash': generate_password_hash(password)}
+ admin = {
+ "is_admin": True,
+ "is_api": True,
+ "is_active": True,
+ "nickname": nickname,
+ "pwdhash": generate_password_hash(password),
+ }
with application.app_context():
UserController(ignore_context=True).create(**admin)
+
@manager.command
def fetch_asyncio(user_id=None, feed_id=None):
"Crawl the feeds with asyncio."
@@ -54,33 +63,32 @@ def fetch_asyncio(user_id=None, feed_id=None):
with application.app_context():
from crawler import default_crawler
+
filters = {}
- filters['is_active'] = True
- filters['automatic_crawling'] = True
+ filters["is_active"] = True
+ filters["automatic_crawling"] = True
if None is not user_id:
- filters['id'] = user_id
+ filters["id"] = user_id
users = UserController().read(**filters).all()
try:
feed_id = int(feed_id)
except:
feed_id = None
-
-
+
loop = asyncio.get_event_loop()
queue = asyncio.Queue(maxsize=3, loop=loop)
-
+
producer_coro = default_crawler.retrieve_feed(queue, users, feed_id)
consumer_coro = default_crawler.insert_articles(queue, 1)
- logger.info('Starting crawler.')
+ logger.info("Starting crawler.")
start = datetime.now()
loop.run_until_complete(asyncio.gather(producer_coro, consumer_coro))
end = datetime.now()
loop.close()
- logger.info('Crawler finished in {} seconds.' \
- .format((end - start).seconds))
+ logger.info("Crawler finished in {} seconds.".format((end - start).seconds))
-if __name__ == '__main__':
+if __name__ == "__main__":
manager.run()
diff --git a/newspipe/notifications/emails.py b/newspipe/notifications/emails.py
index 90c87c93..1d156bd8 100644
--- a/newspipe/notifications/emails.py
+++ b/newspipe/notifications/emails.py
@@ -36,31 +36,33 @@ def send_async_email(mfrom, mto, msg):
s = smtplib.SMTP(conf.NOTIFICATION_HOST)
s.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD)
except Exception:
- logger.exception('send_async_email raised:')
+ logger.exception("send_async_email raised:")
else:
s.sendmail(mfrom, mto, msg.as_string())
s.quit()
+
def send(*args, **kwargs):
"""
This functions enables to send email via different method.
"""
send_smtp(**kwargs)
+
def send_smtp(to="", bcc="", subject="", plaintext="", html=""):
"""
Send an email.
"""
# Create message container - the correct MIME type is multipart/alternative.
- msg = MIMEMultipart('alternative')
- msg['Subject'] = subject
- msg['From'] = conf.NOTIFICATION_EMAIL
- msg['To'] = to
- msg['BCC'] = bcc
+ msg = MIMEMultipart("alternative")
+ msg["Subject"] = subject
+ msg["From"] = conf.NOTIFICATION_EMAIL
+ msg["To"] = to
+ msg["BCC"] = bcc
# Record the MIME types of both parts - text/plain and text/html.
- part1 = MIMEText(plaintext, 'plain', 'utf-8')
- part2 = MIMEText(html, 'html', 'utf-8')
+ part1 = MIMEText(plaintext, "plain", "utf-8")
+ part2 = MIMEText(html, "html", "utf-8")
# Attach parts into message container.
# According to RFC 2046, the last part of a multipart message, in this case
@@ -74,5 +76,7 @@ def send_smtp(to="", bcc="", subject="", plaintext="", html=""):
except Exception:
logger.exception("send_smtp raised:")
else:
- s.sendmail(conf.NOTIFICATION_EMAIL, msg['To'] + ", " + msg['BCC'], msg.as_string())
+ s.sendmail(
+ conf.NOTIFICATION_EMAIL, msg["To"] + ", " + msg["BCC"], msg.as_string()
+ )
s.quit()
diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py
index e775f4b9..2bc24810 100644
--- a/newspipe/notifications/notifications.py
+++ b/newspipe/notifications/notifications.py
@@ -31,23 +31,34 @@ def new_account_notification(user, email):
Account creation notification.
"""
token = generate_confirmation_token(user.nickname)
- expire_time = datetime.datetime.now() + \
- datetime.timedelta(seconds=conf.TOKEN_VALIDITY_PERIOD)
+ expire_time = datetime.datetime.now() + datetime.timedelta(
+ seconds=conf.TOKEN_VALIDITY_PERIOD
+ )
- plaintext = render_template('emails/account_activation.txt',
- user=user, platform_url=conf.PLATFORM_URL,
- token=token,
- expire_time=expire_time)
+ plaintext = render_template(
+ "emails/account_activation.txt",
+ user=user,
+ platform_url=conf.PLATFORM_URL,
+ token=token,
+ expire_time=expire_time,
+ )
+
+ emails.send(
+ to=email,
+ bcc=conf.NOTIFICATION_EMAIL,
+ subject="[Newspipe] Account creation",
+ plaintext=plaintext,
+ )
- emails.send(to=email, bcc=conf.NOTIFICATION_EMAIL,
- subject="[Newspipe] Account creation", plaintext=plaintext)
def new_password_notification(user, password):
"""
New password notification.
"""
- plaintext = render_template('emails/new_password.txt',
- user=user, password=password)
- emails.send(to=user.email,
- bcc=conf.NOTIFICATION_EMAIL,
- subject="[Newspipe] New password", plaintext=plaintext)
+ plaintext = render_template("emails/new_password.txt", user=user, password=password)
+ emails.send(
+ to=user.email,
+ bcc=conf.NOTIFICATION_EMAIL,
+ subject="[Newspipe] New password",
+ plaintext=plaintext,
+ )
diff --git a/newspipe/runserver.py b/newspipe/runserver.py
index a1ebb54c..2b016f1c 100755
--- a/newspipe/runserver.py
+++ b/newspipe/runserver.py
@@ -29,9 +29,11 @@ babel = Babel(application)
# Jinja filters
def month_name(month_number):
return calendar.month_name[month_number]
-application.jinja_env.filters['month_name'] = month_name
-application.jinja_env.filters['datetime'] = format_datetime
-application.jinja_env.globals['conf'] = conf
+
+
+application.jinja_env.filters["month_name"] = month_name
+application.jinja_env.filters["datetime"] = format_datetime
+application.jinja_env.globals["conf"] = conf
# Views
from flask_restful import Api
@@ -39,10 +41,11 @@ from flask import g
with application.app_context():
populate_g()
- g.api = Api(application, prefix='/api/v2.0')
+ g.api = Api(application, prefix="/api/v2.0")
g.babel = babel
from web import views
+
application.register_blueprint(views.articles_bp)
application.register_blueprint(views.article_bp)
application.register_blueprint(views.feeds_bp)
@@ -57,7 +60,7 @@ with application.app_context():
application.register_blueprint(views.bookmark_bp)
-if __name__ == '__main__':
- application.run(host=conf.WEBSERVER_HOST,
- port=conf.WEBSERVER_PORT,
- debug=conf.WEBSERVER_DEBUG)
+if __name__ == "__main__":
+ application.run(
+ host=conf.WEBSERVER_HOST, port=conf.WEBSERVER_PORT, debug=conf.WEBSERVER_DEBUG
+ )
diff --git a/newspipe/web/controllers/__init__.py b/newspipe/web/controllers/__init__.py
index 5fbc2619..9b193cc5 100644
--- a/newspipe/web/controllers/__init__.py
+++ b/newspipe/web/controllers/__init__.py
@@ -7,6 +7,12 @@ from .bookmark import BookmarkController
from .tag import BookmarkTagController
-__all__ = ['FeedController', 'CategoryController', 'ArticleController',
- 'UserController', 'IconController', 'BookmarkController',
- 'BookmarkTagController']
+__all__ = [
+ "FeedController",
+ "CategoryController",
+ "ArticleController",
+ "UserController",
+ "IconController",
+ "BookmarkController",
+ "BookmarkTagController",
+]
diff --git a/newspipe/web/controllers/abstract.py b/newspipe/web/controllers/abstract.py
index 764ff305..9d9e84f2 100644
--- a/newspipe/web/controllers/abstract.py
+++ b/newspipe/web/controllers/abstract.py
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
class AbstractController:
_db_cls = None # reference to the database class
- _user_id_key = 'user_id'
+ _user_id_key = "user_id"
def __init__(self, user_id=None, ignore_context=False):
"""User id is a right management mechanism that should be used to
@@ -36,25 +36,25 @@ class AbstractController:
"""
db_filters = set()
for key, value in filters.items():
- if key == '__or__':
+ if key == "__or__":
db_filters.add(or_(*self._to_filters(**value)))
- elif key.endswith('__gt'):
+ elif key.endswith("__gt"):
db_filters.add(getattr(self._db_cls, key[:-4]) > value)
- elif key.endswith('__lt'):
+ elif key.endswith("__lt"):
db_filters.add(getattr(self._db_cls, key[:-4]) < value)
- elif key.endswith('__ge'):
+ elif key.endswith("__ge"):
db_filters.add(getattr(self._db_cls, key[:-4]) >= value)
- elif key.endswith('__le'):
+ elif key.endswith("__le"):
db_filters.add(getattr(self._db_cls, key[:-4]) <= value)
- elif key.endswith('__ne'):
+ elif key.endswith("__ne"):
db_filters.add(getattr(self._db_cls, key[:-4]) != value)
- elif key.endswith('__in'):
+ elif key.endswith("__in"):
db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
- elif key.endswith('__contains'):
+ elif key.endswith("__contains"):
db_filters.add(getattr(self._db_cls, key[:-10]).contains(value))
- elif key.endswith('__like'):
+ elif key.endswith("__like"):
db_filters.add(getattr(self._db_cls, key[:-6]).like(value))
- elif key.endswith('__ilike'):
+ elif key.endswith("__ilike"):
db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value))
else:
db_filters.add(getattr(self._db_cls, key) == value)
@@ -66,8 +66,11 @@ class AbstractController:
dependent) and the user is not an admin and the filters doesn't already
contains a filter for that user.
"""
- if self._user_id_key is not None and self.user_id \
- and filters.get(self._user_id_key) != self.user_id:
+ if (
+ self._user_id_key is not None
+ and self.user_id
+ and filters.get(self._user_id_key) != self.user_id
+ ):
filters[self._user_id_key] = self.user_id
return self._db_cls.query.filter(*self._to_filters(**filters))
@@ -76,20 +79,27 @@ class AbstractController:
obj = self._get(**filters).first()
if obj and not self._has_right_on(obj):
- raise Forbidden({'message': 'No authorized to access %r (%r)'
- % (self._db_cls.__class__.__name__, filters)})
+ raise Forbidden(
+ {
+ "message": "No authorized to access %r (%r)"
+ % (self._db_cls.__class__.__name__, filters)
+ }
+ )
if not obj:
- raise NotFound({'message': 'No %r (%r)'
- % (self._db_cls.__class__.__name__, filters)})
+ raise NotFound(
+ {"message": "No %r (%r)" % (self._db_cls.__class__.__name__, filters)}
+ )
return obj
def create(self, **attrs):
assert attrs, "attributes to update must not be empty"
if self._user_id_key is not None and self._user_id_key not in attrs:
attrs[self._user_id_key] = self.user_id
- assert self._user_id_key is None or self._user_id_key in attrs \
- or self.user_id is None, \
- "You must provide user_id one way or another"
+ assert (
+ self._user_id_key is None
+ or self._user_id_key in attrs
+ or self.user_id is None
+ ), "You must provide user_id one way or another"
obj = self._db_cls(**attrs)
db.session.add(obj)
@@ -123,39 +133,45 @@ class AbstractController:
# user_id == None is like being admin
if self._user_id_key is None:
return True
- return self.user_id is None \
- or getattr(obj, self._user_id_key, None) == self.user_id
+ return (
+ self.user_id is None
+ or getattr(obj, self._user_id_key, None) == self.user_id
+ )
def _count_by(self, elem_to_group_by, filters):
if self.user_id:
- filters['user_id'] = self.user_id
- return dict(db.session.query(elem_to_group_by, func.count('id'))
- .filter(*self._to_filters(**filters))
- .group_by(elem_to_group_by).all())
+ filters["user_id"] = self.user_id
+ return dict(
+ db.session.query(elem_to_group_by, func.count("id"))
+ .filter(*self._to_filters(**filters))
+ .group_by(elem_to_group_by)
+ .all()
+ )
@classmethod
def _get_attrs_desc(cls, role, right=None):
result = defaultdict(dict)
- if role == 'admin':
+ if role == "admin":
columns = cls._db_cls.__table__.columns.keys()
else:
- assert role in {'base', 'api'}, 'unknown role %r' % role
- assert right in {'read', 'write'}, \
- "right must be 'read' or 'write' with role %r" % role
- columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))()
+ assert role in {"base", "api"}, "unknown role %r" % role
+ assert right in {"read", "write"}, (
+ "right must be 'read' or 'write' with role %r" % role
+ )
+ columns = getattr(cls._db_cls, "fields_%s_%s" % (role, right))()
for column in columns:
result[column] = {}
db_col = getattr(cls._db_cls, column).property.columns[0]
try:
- result[column]['type'] = db_col.type.python_type
+ result[column]["type"] = db_col.type.python_type
except NotImplementedError:
if db_col.default:
- result[column]['type'] = db_col.default.arg.__class__
+ result[column]["type"] = db_col.default.arg.__class__
if column not in result:
continue
- if issubclass(result[column]['type'], datetime):
- result[column]['default'] = datetime.utcnow()
- result[column]['type'] = lambda x: dateutil.parser.parse(x)
+ if issubclass(result[column]["type"], datetime):
+ result[column]["default"] = datetime.utcnow()
+ result[column]["type"] = lambda x: dateutil.parser.parse(x)
elif db_col.default:
- result[column]['default'] = db_col.default.arg
+ result[column]["default"] = db_col.default.arg
return result
diff --git a/newspipe/web/controllers/article.py b/newspipe/web/controllers/article.py
index d7058229..03342a1f 100644
--- a/newspipe/web/controllers/article.py
+++ b/newspipe/web/controllers/article.py
@@ -30,19 +30,24 @@ class ArticleController(AbstractController):
return self._count_by(Article.feed_id, filters)
def count_by_user_id(self, **filters):
- return dict(db.session.query(Article.user_id, func.count(Article.id))
- .filter(*self._to_filters(**filters))
- .group_by(Article.user_id).all())
+ return dict(
+ db.session.query(Article.user_id, func.count(Article.id))
+ .filter(*self._to_filters(**filters))
+ .group_by(Article.user_id)
+ .all()
+ )
def create(self, **attrs):
# handling special denorm for article rights
- assert 'feed_id' in attrs, "must provide feed_id when creating article"
- feed = FeedController(
- attrs.get('user_id', self.user_id)).get(id=attrs['feed_id'])
- if 'user_id' in attrs:
- assert feed.user_id == attrs['user_id'] or self.user_id is None, \
- "no right on feed %r" % feed.id
- attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id
+ assert "feed_id" in attrs, "must provide feed_id when creating article"
+ feed = FeedController(attrs.get("user_id", self.user_id)).get(
+ id=attrs["feed_id"]
+ )
+ if "user_id" in attrs:
+ assert feed.user_id == attrs["user_id"] or self.user_id is None, (
+ "no right on feed %r" % feed.id
+ )
+ attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id
skipped, read, liked = process_filters(feed.filters, attrs)
if skipped:
@@ -51,15 +56,16 @@ class ArticleController(AbstractController):
return article
def update(self, filters, attrs):
- user_id = attrs.get('user_id', self.user_id)
- if 'feed_id' in attrs:
- feed = FeedController().get(id=attrs['feed_id'])
+ user_id = attrs.get("user_id", self.user_id)
+ if "feed_id" in attrs:
+ feed = FeedController().get(id=attrs["feed_id"])
assert feed.user_id == user_id, "no right on feed %r" % feed.id
- attrs['category_id'] = feed.category_id
- if attrs.get('category_id'):
- cat = CategoryController().get(id=attrs['category_id'])
- assert self.user_id is None or cat.user_id == user_id, \
- "no right on cat %r" % cat.id
+ attrs["category_id"] = feed.category_id
+ if attrs.get("category_id"):
+ cat = CategoryController().get(id=attrs["category_id"])
+ assert self.user_id is None or cat.user_id == user_id, (
+ "no right on cat %r" % cat.id
+ )
return super().update(filters, attrs)
def get_history(self, year=None, month=None):
@@ -69,11 +75,11 @@ class ArticleController(AbstractController):
articles_counter = Counter()
articles = self.read()
if year is not None:
- articles = articles.filter(
- sqlalchemy.extract('year', Article.date) == year)
+ articles = articles.filter(sqlalchemy.extract("year", Article.date) == year)
if month is not None:
articles = articles.filter(
- sqlalchemy.extract('month', Article.date) == month)
+ sqlalchemy.extract("month", Article.date) == month
+ )
for article in articles.all():
if year is not None:
articles_counter[article.date.month] += 1
@@ -82,6 +88,17 @@ class ArticleController(AbstractController):
return articles_counter, articles
def read_light(self, **filters):
- return super().read(**filters).with_entities(Article.id, Article.title,
- Article.readed, Article.like, Article.feed_id, Article.date,
- Article.category_id).order_by(Article.date.desc())
+ return (
+ super()
+ .read(**filters)
+ .with_entities(
+ Article.id,
+ Article.title,
+ Article.readed,
+ Article.like,
+ Article.feed_id,
+ Article.date,
+ Article.category_id,
+ )
+ .order_by(Article.date.desc())
+ )
diff --git a/newspipe/web/controllers/bookmark.py b/newspipe/web/controllers/bookmark.py
index b5413243..d1c1260c 100644
--- a/newspipe/web/controllers/bookmark.py
+++ b/newspipe/web/controllers/bookmark.py
@@ -17,16 +17,19 @@ class BookmarkController(AbstractController):
return self._count_by(Bookmark.href, filters)
def update(self, filters, attrs):
- BookmarkTagController(self.user_id) \
- .read(**{'bookmark_id': filters["id"]}) \
- .delete()
+ BookmarkTagController(self.user_id).read(
+ **{"bookmark_id": filters["id"]}
+ ).delete()
- for tag in attrs['tags']:
+ for tag in attrs["tags"]:
BookmarkTagController(self.user_id).create(
- **{'text': tag.text,
- 'id': tag.id,
- 'bookmark_id': tag.bookmark_id,
- 'user_id': tag.user_id})
-
- del attrs['tags']
+ **{
+ "text": tag.text,
+ "id": tag.id,
+ "bookmark_id": tag.bookmark_id,
+ "user_id": tag.user_id,
+ }
+ )
+
+ del attrs["tags"]
return super().update(filters, attrs)
diff --git a/newspipe/web/controllers/category.py b/newspipe/web/controllers/category.py
index fef5ca81..ec54f5a3 100644
--- a/newspipe/web/controllers/category.py
+++ b/newspipe/web/controllers/category.py
@@ -7,6 +7,7 @@ class CategoryController(AbstractController):
_db_cls = Category
def delete(self, obj_id):
- FeedController(self.user_id).update({'category_id': obj_id},
- {'category_id': None})
+ FeedController(self.user_id).update(
+ {"category_id": obj_id}, {"category_id": None}
+ )
return super().delete(obj_id)
diff --git a/newspipe/web/controllers/feed.py b/newspipe/web/controllers/feed.py
index d75cd994..19ba463f 100644
--- a/newspipe/web/controllers/feed.py
+++ b/newspipe/web/controllers/feed.py
@@ -16,22 +16,26 @@ DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR
class FeedController(AbstractController):
_db_cls = Feed
- def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR,
- limit=DEFAULT_LIMIT):
- return [feed for feed in self.read(
- error_count__lt=max_error, enabled=True,
- last_retrieved__lt=max_last)
- .join(User).filter(User.is_active == True)
- .order_by('last_retrieved')
- .limit(limit)]
+ def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
+ return [
+ feed
+ for feed in self.read(
+ error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last
+ )
+ .join(User)
+ .filter(User.is_active == True)
+ .order_by("last_retrieved")
+ .limit(limit)
+ ]
def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
now = datetime.now()
max_last = now - timedelta(minutes=60)
feeds = self.list_late(max_last, max_error, limit)
if feeds:
- self.update({'id__in': [feed.id for feed in feeds]},
- {'last_retrieved': now})
+ self.update(
+ {"id__in": [feed.id for feed in feeds]}, {"last_retrieved": now}
+ )
return feeds
def get_duplicates(self, feed_id):
@@ -43,8 +47,9 @@ class FeedController(AbstractController):
duplicates = []
for pair in itertools.combinations(feed.articles[:1000], 2):
date1, date2 = pair[0].date, pair[1].date
- if clear_string(pair[0].title) == clear_string(pair[1].title) \
- and (date1 - date2) < timedelta(days=1):
+ if clear_string(pair[0].title) == clear_string(pair[1].title) and (
+ date1 - date2
+ ) < timedelta(days=1):
if pair[0].retrieved_date < pair[1].retrieved_date:
duplicates.append((pair[0], pair[1]))
else:
@@ -75,11 +80,11 @@ class FeedController(AbstractController):
return self._count_by(Feed.link, filters)
def _ensure_icon(self, attrs):
- if not attrs.get('icon_url'):
+ if not attrs.get("icon_url"):
return
icon_contr = IconController()
- if not icon_contr.read(url=attrs['icon_url']).count():
- icon_contr.create(**{'url': attrs['icon_url']})
+ if not icon_contr.read(url=attrs["icon_url"]).count():
+ icon_contr.create(**{"url": attrs["icon_url"]})
def create(self, **attrs):
self._ensure_icon(attrs)
@@ -87,12 +92,14 @@ class FeedController(AbstractController):
def update(self, filters, attrs):
from .article import ArticleController
+
self._ensure_icon(attrs)
- if 'category_id' in attrs and attrs['category_id'] == 0:
- del attrs['category_id']
- elif 'category_id' in attrs:
+ if "category_id" in attrs and attrs["category_id"] == 0:
+ del attrs["category_id"]
+ elif "category_id" in attrs:
art_contr = ArticleController(self.user_id)
for feed in self.read(**filters):
- art_contr.update({'feed_id': feed.id},
- {'category_id': attrs['category_id']})
+ art_contr.update(
+ {"feed_id": feed.id}, {"category_id": attrs["category_id"]}
+ )
return super().update(filters, attrs)
diff --git a/newspipe/web/controllers/icon.py b/newspipe/web/controllers/icon.py
index 07c4a4ef..de86b52f 100644
--- a/newspipe/web/controllers/icon.py
+++ b/newspipe/web/controllers/icon.py
@@ -9,11 +9,15 @@ class IconController(AbstractController):
_user_id_key = None
def _build_from_url(self, attrs):
- if 'url' in attrs and 'content' not in attrs:
- resp = requests.get(attrs['url'], verify=False)
- attrs.update({'url': resp.url,
- 'mimetype': resp.headers.get('content-type', None),
- 'content': base64.b64encode(resp.content).decode('utf8')})
+ if "url" in attrs and "content" not in attrs:
+ resp = requests.get(attrs["url"], verify=False)
+ attrs.update(
+ {
+ "url": resp.url,
+ "mimetype": resp.headers.get("content-type", None),
+ "content": base64.b64encode(resp.content).decode("utf8"),
+ }
+ )
return attrs
def create(self, **attrs):
diff --git a/newspipe/web/controllers/user.py b/newspipe/web/controllers/user.py
index 6ab04d44..71eb7d08 100644
--- a/newspipe/web/controllers/user.py
+++ b/newspipe/web/controllers/user.py
@@ -8,13 +8,13 @@ logger = logging.getLogger(__name__)
class UserController(AbstractController):
_db_cls = User
- _user_id_key = 'id'
+ _user_id_key = "id"
def _handle_password(self, attrs):
- if attrs.get('password'):
- attrs['pwdhash'] = generate_password_hash(attrs.pop('password'))
- elif 'password' in attrs:
- del attrs['password']
+ if attrs.get("password"):
+ attrs["pwdhash"] = generate_password_hash(attrs.pop("password"))
+ elif "password" in attrs:
+ del attrs["password"]
def check_password(self, user, password):
return check_password_hash(user.pwdhash, password)
diff --git a/newspipe/web/decorators.py b/newspipe/web/decorators.py
index 3835f646..8569a024 100644
--- a/newspipe/web/decorators.py
+++ b/newspipe/web/decorators.py
@@ -13,9 +13,11 @@ def async_maker(f):
indexing the database) in background.
This prevent the server to freeze.
"""
+
def wrapper(*args, **kwargs):
thr = Thread(target=f, args=args, kwargs=kwargs)
thr.start()
+
return wrapper
@@ -24,4 +26,5 @@ def pyagg_default_decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
+
return wrapper
diff --git a/newspipe/web/forms.py b/newspipe/web/forms.py
index 7b1893e2..046ce1ec 100644
--- a/newspipe/web/forms.py
+++ b/newspipe/web/forms.py
@@ -30,8 +30,17 @@ from flask import flash, url_for, redirect
from flask_wtf import FlaskForm
from flask_babel import lazy_gettext
from werkzeug.exceptions import NotFound
-from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \
- SubmitField, IntegerField, SelectField, validators, HiddenField
+from wtforms import (
+ TextField,
+ TextAreaField,
+ PasswordField,
+ BooleanField,
+ SubmitField,
+ IntegerField,
+ SelectField,
+ validators,
+ HiddenField,
+)
from wtforms.fields.html5 import EmailField, URLField
from lib import misc_utils
@@ -43,27 +52,44 @@ class SignupForm(FlaskForm):
"""
Sign up form (registration to newspipe).
"""
- nickname = TextField(lazy_gettext("Nickname"),
- [validators.Required(lazy_gettext("Please enter your nickname."))])
- email = EmailField(lazy_gettext("Email"),
- [validators.Length(min=6, max=35),
- validators.Required(
- lazy_gettext("Please enter your email address (only for account activation, won't be stored)."))])
- password = PasswordField(lazy_gettext("Password"),
- [validators.Required(lazy_gettext("Please enter a password.")),
- validators.Length(min=6, max=100)])
+
+ nickname = TextField(
+ lazy_gettext("Nickname"),
+ [validators.Required(lazy_gettext("Please enter your nickname."))],
+ )
+ email = EmailField(
+ lazy_gettext("Email"),
+ [
+ validators.Length(min=6, max=35),
+ validators.Required(
+ lazy_gettext(
+ "Please enter your email address (only for account activation, won't be stored)."
+ )
+ ),
+ ],
+ )
+ password = PasswordField(
+ lazy_gettext("Password"),
+ [
+ validators.Required(lazy_gettext("Please enter a password.")),
+ validators.Length(min=6, max=100),
+ ],
+ )
submit = SubmitField(lazy_gettext("Sign up"))
def validate(self):
ucontr = UserController()
validated = super().validate()
if ucontr.read(nickname=self.nickname.data).count():
- self.nickname.errors.append('Nickname already taken')
+ self.nickname.errors.append("Nickname already taken")
validated = False
if self.nickname.data != User.make_valid_nickname(self.nickname.data):
- self.nickname.errors.append(lazy_gettext(
- 'This nickname has invalid characters. '
- 'Please use letters, numbers, dots and underscores only.'))
+ self.nickname.errors.append(
+ lazy_gettext(
+ "This nickname has invalid characters. "
+ "Please use letters, numbers, dots and underscores only."
+ )
+ )
validated = False
return validated
@@ -72,14 +98,15 @@ class RedirectForm(FlaskForm):
"""
Secure back redirects with WTForms.
"""
+
next = HiddenField()
def __init__(self, *args, **kwargs):
FlaskForm.__init__(self, *args, **kwargs)
if not self.next.data:
- self.next.data = misc_utils.get_redirect_target() or ''
+ self.next.data = misc_utils.get_redirect_target() or ""
- def redirect(self, endpoint='home', **values):
+ def redirect(self, endpoint="home", **values):
if misc_utils.is_safe_url(self.next.data):
return redirect(self.next.data)
target = misc_utils.get_redirect_target()
@@ -90,13 +117,21 @@ class SigninForm(RedirectForm):
"""
Sign in form (connection to newspipe).
"""
- nickmane = TextField("Nickname",
- [validators.Length(min=3, max=35),
- validators.Required(
- lazy_gettext("Please enter your nickname."))])
- password = PasswordField(lazy_gettext('Password'),
- [validators.Required(lazy_gettext("Please enter a password.")),
- validators.Length(min=6, max=100)])
+
+ nickmane = TextField(
+ "Nickname",
+ [
+ validators.Length(min=3, max=35),
+ validators.Required(lazy_gettext("Please enter your nickname.")),
+ ],
+ )
+ password = PasswordField(
+ lazy_gettext("Password"),
+ [
+ validators.Required(lazy_gettext("Please enter a password.")),
+ validators.Length(min=6, max=100),
+ ],
+ )
submit = SubmitField(lazy_gettext("Log In"))
def __init__(self, *args, **kwargs):
@@ -109,15 +144,14 @@ class SigninForm(RedirectForm):
try:
user = ucontr.get(nickname=self.nickmane.data)
except NotFound:
- self.nickmane.errors.append(
- 'Wrong nickname')
+ self.nickmane.errors.append("Wrong nickname")
validated = False
else:
if not user.is_active:
- self.nickmane.errors.append('Account not active')
+ self.nickmane.errors.append("Account not active")
validated = False
if not ucontr.check_password(user, self.password.data):
- self.password.errors.append('Wrong password')
+ self.password.errors.append("Wrong password")
validated = False
self.user = user
return validated
@@ -127,19 +161,24 @@ class UserForm(FlaskForm):
"""
Create or edit a user (for the administrator).
"""
- nickname = TextField(lazy_gettext("Nickname"),
- [validators.Required(lazy_gettext("Please enter your nickname."))])
+
+ nickname = TextField(
+ lazy_gettext("Nickname"),
+ [validators.Required(lazy_gettext("Please enter your nickname."))],
+ )
password = PasswordField(lazy_gettext("Password"))
- automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"),
- default=True)
+ automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True)
submit = SubmitField(lazy_gettext("Save"))
def validate(self):
validated = super(UserForm, self).validate()
if self.nickname.data != User.make_valid_nickname(self.nickname.data):
- self.nickname.errors.append(lazy_gettext(
- 'This nickname has invalid characters. '
- 'Please use letters, numbers, dots and underscores only.'))
+ self.nickname.errors.append(
+ lazy_gettext(
+ "This nickname has invalid characters. "
+ "Please use letters, numbers, dots and underscores only."
+ )
+ )
validated = False
return validated
@@ -148,17 +187,18 @@ class ProfileForm(FlaskForm):
"""
Edit user information.
"""
- nickname = TextField(lazy_gettext("Nickname"),
- [validators.Required(lazy_gettext("Please enter your nickname."))])
+
+ nickname = TextField(
+ lazy_gettext("Nickname"),
+ [validators.Required(lazy_gettext("Please enter your nickname."))],
+ )
password = PasswordField(lazy_gettext("Password"))
password_conf = PasswordField(lazy_gettext("Password Confirmation"))
- automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"),
- default=True)
+ automatic_crawling = BooleanField(lazy_gettext("Automatic crawling"), default=True)
bio = TextAreaField(lazy_gettext("Bio"))
webpage = URLField(lazy_gettext("Webpage"))
twitter = URLField(lazy_gettext("Twitter"))
- is_public_profile = BooleanField(lazy_gettext("Public profile"),
- default=True)
+ is_public_profile = BooleanField(lazy_gettext("Public profile"), default=True)
submit = SubmitField(lazy_gettext("Save"))
def validate(self):
@@ -169,28 +209,34 @@ class ProfileForm(FlaskForm):
self.password_conf.errors.append(message)
validated = False
if self.nickname.data != User.make_valid_nickname(self.nickname.data):
- self.nickname.errors.append(lazy_gettext('This nickname has '
- 'invalid characters. Please use letters, numbers, dots and'
- ' underscores only.'))
+ self.nickname.errors.append(
+ lazy_gettext(
+ "This nickname has "
+ "invalid characters. Please use letters, numbers, dots and"
+ " underscores only."
+ )
+ )
validated = False
return validated
class AddFeedForm(FlaskForm):
title = TextField(lazy_gettext("Title"), [validators.Optional()])
- link = TextField(lazy_gettext("Feed link"),
- [validators.Required(lazy_gettext("Please enter the URL."))])
+ link = TextField(
+ lazy_gettext("Feed link"),
+ [validators.Required(lazy_gettext("Please enter the URL."))],
+ )
site_link = TextField(lazy_gettext("Site link"), [validators.Optional()])
enabled = BooleanField(lazy_gettext("Check for updates"), default=True)
submit = SubmitField(lazy_gettext("Save"))
- category_id = SelectField(lazy_gettext("Category of the feed"),
- [validators.Optional()])
+ category_id = SelectField(
+ lazy_gettext("Category of the feed"), [validators.Optional()]
+ )
private = BooleanField(lazy_gettext("Private"), default=False)
def set_category_choices(self, categories):
- self.category_id.choices = [('0', 'No Category')]
- self.category_id.choices += [(str(cat.id), cat.name)
- for cat in categories]
+ self.category_id.choices = [("0", "No Category")]
+ self.category_id.choices += [(str(cat.id), cat.name) for cat in categories]
class CategoryForm(FlaskForm):
@@ -199,13 +245,13 @@ class CategoryForm(FlaskForm):
class BookmarkForm(FlaskForm):
- href = TextField(lazy_gettext("URL"),
- [validators.Required(
- lazy_gettext("Please enter an URL."))])
- title = TextField(lazy_gettext("Title"),
- [validators.Length(min=0, max=100)])
- description = TextField(lazy_gettext("Description"),
- [validators.Length(min=0, max=500)])
+ href = TextField(
+ lazy_gettext("URL"), [validators.Required(lazy_gettext("Please enter an URL."))]
+ )
+ title = TextField(lazy_gettext("Title"), [validators.Length(min=0, max=100)])
+ description = TextField(
+ lazy_gettext("Description"), [validators.Length(min=0, max=500)]
+ )
tags = TextField(lazy_gettext("Tags"))
to_read = BooleanField(lazy_gettext("To read"), default=False)
shared = BooleanField(lazy_gettext("Shared"), default=False)
@@ -213,8 +259,12 @@ class BookmarkForm(FlaskForm):
class InformationMessageForm(FlaskForm):
- subject = TextField(lazy_gettext("Subject"),
- [validators.Required(lazy_gettext("Please enter a subject."))])
- message = TextAreaField(lazy_gettext("Message"),
- [validators.Required(lazy_gettext("Please enter a content."))])
+ subject = TextField(
+ lazy_gettext("Subject"),
+ [validators.Required(lazy_gettext("Please enter a subject."))],
+ )
+ message = TextAreaField(
+ lazy_gettext("Message"),
+ [validators.Required(lazy_gettext("Please enter a content."))],
+ )
submit = SubmitField(lazy_gettext("Send"))
diff --git a/newspipe/web/lib/user_utils.py b/newspipe/web/lib/user_utils.py
index f78a6ed6..84b1c75c 100644
--- a/newspipe/web/lib/user_utils.py
+++ b/newspipe/web/lib/user_utils.py
@@ -1,22 +1,20 @@
-
-
from itsdangerous import URLSafeTimedSerializer
import conf
from bootstrap import application
def generate_confirmation_token(nickname):
- serializer = URLSafeTimedSerializer(application.config['SECRET_KEY'])
- return serializer.dumps(nickname, salt=application.config['SECURITY_PASSWORD_SALT'])
+ serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"])
+ return serializer.dumps(nickname, salt=application.config["SECURITY_PASSWORD_SALT"])
def confirm_token(token):
- serializer = URLSafeTimedSerializer(application.config['SECRET_KEY'])
+ serializer = URLSafeTimedSerializer(application.config["SECRET_KEY"])
try:
nickname = serializer.loads(
token,
- salt=application.config['SECURITY_PASSWORD_SALT'],
- max_age=conf.TOKEN_VALIDITY_PERIOD
+ salt=application.config["SECURITY_PASSWORD_SALT"],
+ max_age=conf.TOKEN_VALIDITY_PERIOD,
)
except:
return False
diff --git a/newspipe/web/lib/view_utils.py b/newspipe/web/lib/view_utils.py
index 1d8c6aed..218ebb4c 100644
--- a/newspipe/web/lib/view_utils.py
+++ b/newspipe/web/lib/view_utils.py
@@ -15,12 +15,14 @@ def etag_match(func):
headers = {}
else:
return response
- if request.headers.get('if-none-match') == etag:
+ if request.headers.get("if-none-match") == etag:
response = Response(status=304)
- response.headers['Cache-Control'] \
- = headers.get('Cache-Control', 'pragma: no-cache')
+ response.headers["Cache-Control"] = headers.get(
+ "Cache-Control", "pragma: no-cache"
+ )
elif not isinstance(response, Response):
response = make_response(response)
- response.headers['etag'] = etag
+ response.headers["etag"] = etag
return response
+
return wrapper
diff --git a/newspipe/web/models/__init__.py b/newspipe/web/models/__init__.py
index bfb1368c..09249368 100644
--- a/newspipe/web/models/__init__.py
+++ b/newspipe/web/models/__init__.py
@@ -36,18 +36,29 @@ from .tag import BookmarkTag
from .tag import ArticleTag
from .bookmark import Bookmark
-__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category',
- 'Bookmark', 'ArticleTag', 'BookmarkTag']
+__all__ = [
+ "Feed",
+ "Role",
+ "User",
+ "Article",
+ "Icon",
+ "Category",
+ "Bookmark",
+ "ArticleTag",
+ "BookmarkTag",
+]
import os
from sqlalchemy.engine import reflection
from sqlalchemy.schema import (
- MetaData,
- Table,
- DropTable,
- ForeignKeyConstraint,
- DropConstraint)
+ MetaData,
+ Table,
+ DropTable,
+ ForeignKeyConstraint,
+ DropConstraint,
+)
+
def db_empty(db):
"Will drop every datas stocked in db."
@@ -71,9 +82,9 @@ def db_empty(db):
for table_name in inspector.get_table_names():
fks = []
for fk in inspector.get_foreign_keys(table_name):
- if not fk['name']:
+ if not fk["name"]:
continue
- fks.append(ForeignKeyConstraint((), (), name=fk['name']))
+ fks.append(ForeignKeyConstraint((), (), name=fk["name"]))
t = Table(table_name, metadata, *fks)
tbs.append(t)
all_fks.extend(fks)
diff --git a/newspipe/web/models/article.py b/newspipe/web/models/article.py
index d55e59c1..c826a3c9 100644
--- a/newspipe/web/models/article.py
+++ b/newspipe/web/models/article.py
@@ -48,40 +48,54 @@ class Article(db.Model, RightMixin):
retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow)
# foreign keys
- user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
- feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id'))
- category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
+ user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
+ feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id"))
+ category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))
# relationships
- tag_objs = db.relationship('ArticleTag', back_populates='article',
- cascade='all,delete-orphan',
- lazy=False,
- foreign_keys='[ArticleTag.article_id]')
- tags = association_proxy('tag_objs', 'text')
+ tag_objs = db.relationship(
+ "ArticleTag",
+ back_populates="article",
+ cascade="all,delete-orphan",
+ lazy=False,
+ foreign_keys="[ArticleTag.article_id]",
+ )
+ tags = association_proxy("tag_objs", "text")
# indexes
- #__table_args__ = (
+ # __table_args__ = (
# Index('user_id'),
# Index('user_id', 'category_id'),
# Index('user_id', 'feed_id'),
# Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id)
- #)
+ # )
# api whitelists
@staticmethod
def _fields_base_write():
- return {'readed', 'like', 'feed_id', 'category_id'}
+ return {"readed", "like", "feed_id", "category_id"}
@staticmethod
def _fields_base_read():
- return {'id', 'entry_id', 'link', 'title', 'content', 'date',
- 'retrieved_date', 'user_id', 'tags'}
+ return {
+ "id",
+ "entry_id",
+ "link",
+ "title",
+ "content",
+ "date",
+ "retrieved_date",
+ "user_id",
+ "tags",
+ }
@staticmethod
def _fields_api_write():
- return {'tags'}
+ return {"tags"}
def __repr__(self):
- return "<Article(id=%d, entry_id=%s, title=%r, " \
- "date=%r, retrieved_date=%r)>" % (self.id, self.entry_id,
- self.title, self.date, self.retrieved_date)
+ return (
+ "<Article(id=%d, entry_id=%s, title=%r, "
+ "date=%r, retrieved_date=%r)>"
+ % (self.id, self.entry_id, self.title, self.date, self.retrieved_date)
+ )
diff --git a/newspipe/web/models/bookmark.py b/newspipe/web/models/bookmark.py
index eb6b73e3..c557225e 100644
--- a/newspipe/web/models/bookmark.py
+++ b/newspipe/web/models/bookmark.py
@@ -40,6 +40,7 @@ class Bookmark(db.Model, RightMixin):
"""
Represent a bookmark.
"""
+
id = db.Column(db.Integer(), primary_key=True)
href = db.Column(db.String(), default="")
title = db.Column(db.String(), default="")
@@ -47,22 +48,25 @@ class Bookmark(db.Model, RightMixin):
shared = db.Column(db.Boolean(), default=False)
to_read = db.Column(db.Boolean(), default=False)
time = db.Column(db.DateTime(), default=datetime.utcnow)
- user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+ user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
# relationships
- tags = db.relationship(BookmarkTag, backref='of_bookmark', lazy='dynamic',
- cascade='all,delete-orphan',
- order_by=desc(BookmarkTag.text))
- tags_proxy = association_proxy('tags', 'text')
-
+ tags = db.relationship(
+ BookmarkTag,
+ backref="of_bookmark",
+ lazy="dynamic",
+ cascade="all,delete-orphan",
+ order_by=desc(BookmarkTag.text),
+ )
+ tags_proxy = association_proxy("tags", "text")
- @validates('description')
+ @validates("description")
def validates_title(self, key, value):
return str(value).strip()
- @validates('extended')
+ @validates("extended")
def validates_description(self, key, value):
return str(value).strip()
def __repr__(self):
- return '<Bookmark %r>' % (self.href)
+ return "<Bookmark %r>" % (self.href)
diff --git a/newspipe/web/models/category.py b/newspipe/web/models/category.py
index 2da7809a..bb47ce45 100644
--- a/newspipe/web/models/category.py
+++ b/newspipe/web/models/category.py
@@ -11,19 +11,18 @@ class Category(db.Model, RightMixin):
name = db.Column(db.String())
# relationships
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
- feeds = db.relationship('Feed', cascade='all,delete-orphan')
- articles = db.relationship('Article',
- cascade='all,delete-orphan')
+ user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
+ feeds = db.relationship("Feed", cascade="all,delete-orphan")
+ articles = db.relationship("Article", cascade="all,delete-orphan")
# index
- idx_category_uid = Index('user_id')
+ idx_category_uid = Index("user_id")
# api whitelists
@staticmethod
def _fields_base_read():
- return {'id', 'user_id'}
+ return {"id", "user_id"}
@staticmethod
def _fields_base_write():
- return {'name'}
+ return {"name"}
diff --git a/newspipe/web/models/feed.py b/newspipe/web/models/feed.py
index fc0b64cb..f440a39c 100644
--- a/newspipe/web/models/feed.py
+++ b/newspipe/web/models/feed.py
@@ -38,6 +38,7 @@ class Feed(db.Model, RightMixin):
"""
Represent a feed.
"""
+
id = db.Column(db.Integer(), primary_key=True)
title = db.Column(db.String(), default="")
description = db.Column(db.String(), default="FR")
@@ -58,34 +59,47 @@ class Feed(db.Model, RightMixin):
error_count = db.Column(db.Integer(), default=0)
# relationship
- icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
- user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
- category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
- articles = db.relationship(Article, backref='source', lazy='dynamic',
- cascade='all,delete-orphan',
- order_by=desc(Article.date))
+ icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None)
+ user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
+ category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))
+ articles = db.relationship(
+ Article,
+ backref="source",
+ lazy="dynamic",
+ cascade="all,delete-orphan",
+ order_by=desc(Article.date),
+ )
# index
- idx_feed_uid_cid = Index('user_id', 'category_id')
- idx_feed_uid = Index('user_id')
+ idx_feed_uid_cid = Index("user_id", "category_id")
+ idx_feed_uid = Index("user_id")
- # api whitelists
+ # api whitelists
@staticmethod
def _fields_base_write():
- return {'title', 'description', 'link', 'site_link', 'enabled',
- 'filters', 'last_error', 'error_count', 'category_id'}
+ return {
+ "title",
+ "description",
+ "link",
+ "site_link",
+ "enabled",
+ "filters",
+ "last_error",
+ "error_count",
+ "category_id",
+ }
@staticmethod
def _fields_base_read():
- return {'id', 'user_id', 'icon_url', 'last_retrieved'}
+ return {"id", "user_id", "icon_url", "last_retrieved"}
- @validates('title')
+ @validates("title")
def validates_title(self, key, value):
return str(value).strip()
- @validates('description')
+ @validates("description")
def validates_description(self, key, value):
return str(value).strip()
def __repr__(self):
- return '<Feed %r>' % (self.title)
+ return "<Feed %r>" % (self.title)
diff --git a/newspipe/web/models/right_mixin.py b/newspipe/web/models/right_mixin.py
index 1c316f95..670beafa 100644
--- a/newspipe/web/models/right_mixin.py
+++ b/newspipe/web/models/right_mixin.py
@@ -2,14 +2,13 @@ from sqlalchemy.ext.associationproxy import _AssociationList
class RightMixin:
-
@staticmethod
def _fields_base_write():
return set()
@staticmethod
def _fields_base_read():
- return set(['id'])
+ return set(["id"])
@staticmethod
def _fields_api_write():
@@ -17,7 +16,7 @@ class RightMixin:
@staticmethod
def _fields_api_read():
- return set(['id'])
+ return set(["id"])
@classmethod
def fields_base_write(cls):
@@ -36,26 +35,28 @@ class RightMixin:
return cls.fields_base_read().union(cls._fields_api_read())
def __getitem__(self, key):
- if not hasattr(self, '__dump__'):
+ if not hasattr(self, "__dump__"):
self.__dump__ = {}
return self.__dump__.get(key)
def __setitem__(self, key, value):
- if not hasattr(self, '__dump__'):
+ if not hasattr(self, "__dump__"):
self.__dump__ = {}
self.__dump__[key] = value
- def dump(self, role='admin'):
- if role == 'admin':
- dico = {k: getattr(self, k)
- for k in set(self.__table__.columns.keys())
- .union(self.fields_api_read())
- .union(self.fields_base_read())}
- elif role == 'api':
+ def dump(self, role="admin"):
+ if role == "admin":
+ dico = {
+ k: getattr(self, k)
+ for k in set(self.__table__.columns.keys())
+ .union(self.fields_api_read())
+ .union(self.fields_base_read())
+ }
+ elif role == "api":
dico = {k: getattr(self, k) for k in self.fields_api_read()}
else:
dico = {k: getattr(self, k) for k in self.fields_base_read()}
- if hasattr(self, '__dump__'):
+ if hasattr(self, "__dump__"):
dico.update(self.__dump__)
for key, value in dico.items(): # preventing association proxy to die
if isinstance(value, _AssociationList):
diff --git a/newspipe/web/models/role.py b/newspipe/web/models/role.py
index 0a2ecd4a..15a08b73 100644
--- a/newspipe/web/models/role.py
+++ b/newspipe/web/models/role.py
@@ -33,7 +33,8 @@ class Role(db.Model):
"""
Represent a role.
"""
+
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(), unique=True)
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+ user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
diff --git a/newspipe/web/models/tag.py b/newspipe/web/models/tag.py
index 76467c0b..9bd8afc5 100644
--- a/newspipe/web/models/tag.py
+++ b/newspipe/web/models/tag.py
@@ -8,12 +8,14 @@ class ArticleTag(db.Model):
text = db.Column(db.String, primary_key=True, unique=False)
# foreign keys
- article_id = db.Column(db.Integer, db.ForeignKey('article.id', ondelete='CASCADE'),
- primary_key=True)
+ article_id = db.Column(
+ db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True
+ )
# relationships
- article = db.relationship('Article', back_populates='tag_objs',
- foreign_keys=[article_id])
+ article = db.relationship(
+ "Article", back_populates="tag_objs", foreign_keys=[article_id]
+ )
def __init__(self, text):
self.text = text
@@ -24,12 +26,18 @@ class BookmarkTag(db.Model):
text = db.Column(db.String, unique=False)
# foreign keys
- user_id = db.Column(db.Integer, db.ForeignKey('user.id', ondelete='CASCADE'))
- bookmark_id = db.Column(db.Integer, db.ForeignKey('bookmark.id', ondelete='CASCADE'))
+ user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE"))
+ bookmark_id = db.Column(
+ db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE")
+ )
# relationships
- bookmark = db.relationship('Bookmark', back_populates='tags',
- cascade="all,delete", foreign_keys=[bookmark_id])
+ bookmark = db.relationship(
+ "Bookmark",
+ back_populates="tags",
+ cascade="all,delete",
+ foreign_keys=[bookmark_id],
+ )
# def __init__(self, text, user_id):
# self.text = text
diff --git a/newspipe/web/models/user.py b/newspipe/web/models/user.py
index 4d65c3c5..a088c74e 100644
--- a/newspipe/web/models/user.py
+++ b/newspipe/web/models/user.py
@@ -44,6 +44,7 @@ class User(db.Model, UserMixin, RightMixin):
"""
Represent a user.
"""
+
id = db.Column(db.Integer, primary_key=True)
nickname = db.Column(db.String(), unique=True)
pwdhash = db.Column(db.String())
@@ -64,29 +65,34 @@ class User(db.Model, UserMixin, RightMixin):
is_api = db.Column(db.Boolean(), default=False)
# relationships
- categories = db.relationship('Category', backref='user',
- cascade='all, delete-orphan',
- foreign_keys=[Category.user_id])
- feeds = db.relationship('Feed', backref='user',
- cascade='all, delete-orphan',
- foreign_keys=[Feed.user_id])
+ categories = db.relationship(
+ "Category",
+ backref="user",
+ cascade="all, delete-orphan",
+ foreign_keys=[Category.user_id],
+ )
+ feeds = db.relationship(
+ "Feed",
+ backref="user",
+ cascade="all, delete-orphan",
+ foreign_keys=[Feed.user_id],
+ )
@staticmethod
def _fields_base_write():
- return {'login', 'password'}
+ return {"login", "password"}
@staticmethod
def _fields_base_read():
- return {'date_created', 'last_connection'}
+ return {"date_created", "last_connection"}
@staticmethod
def make_valid_nickname(nickname):
- return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
+ return re.sub("[^a-zA-Z0-9_\.]", "", nickname)
- @validates('bio')
+ @validates("bio")
def validates_bio(self, key, value):
- assert len(value) <= 5000, \
- AssertionError("maximum length for bio: 5000")
+ assert len(value) <= 5000, AssertionError("maximum length for bio: 5000")
return value.strip()
def get_id(self):
@@ -105,4 +111,4 @@ class User(db.Model, UserMixin, RightMixin):
return self.id == other.id
def __repr__(self):
- return '<User %r>' % (self.nickname)
+ return "<User %r>" % (self.nickname)
diff --git a/newspipe/web/views/__init__.py b/newspipe/web/views/__init__.py
index 41bb52f3..b8b415f9 100644
--- a/newspipe/web/views/__init__.py
+++ b/newspipe/web/views/__init__.py
@@ -8,10 +8,25 @@ from web.views.admin import admin_bp
from web.views.user import user_bp, users_bp
from web.views.bookmark import bookmark_bp, bookmarks_bp
-__all__ = ['views', 'home', 'session_mgmt', 'v2', 'v3',
- 'article_bp', 'articles_bp', 'feed_bp', 'feeds_bp',
- 'category_bp', 'categories_bp', 'icon_bp',
- 'admin_bp', 'user_bp', 'users_bp', 'bookmark_bp', 'bookmarks_bp']
+__all__ = [
+ "views",
+ "home",
+ "session_mgmt",
+ "v2",
+ "v3",
+ "article_bp",
+ "articles_bp",
+ "feed_bp",
+ "feeds_bp",
+ "category_bp",
+ "categories_bp",
+ "icon_bp",
+ "admin_bp",
+ "user_bp",
+ "users_bp",
+ "bookmark_bp",
+ "bookmarks_bp",
+]
import conf
from flask import request
diff --git a/newspipe/web/views/admin.py b/newspipe/web/views/admin.py
index 73b2b668..fe1b389b 100644
--- a/newspipe/web/views/admin.py
+++ b/newspipe/web/views/admin.py
@@ -1,5 +1,5 @@
from datetime import datetime
-from flask import (Blueprint, render_template, redirect, flash, url_for)
+from flask import Blueprint, render_template, redirect, flash, url_for
from flask_babel import gettext, format_timedelta
from flask_login import login_required, current_user
@@ -8,41 +8,45 @@ from web.views.common import admin_permission
from web.controllers import UserController
from web.forms import InformationMessageForm, UserForm
-admin_bp = Blueprint('admin', __name__, url_prefix='/admin')
+admin_bp = Blueprint("admin", __name__, url_prefix="/admin")
-@admin_bp.route('/dashboard', methods=['GET', 'POST'])
+@admin_bp.route("/dashboard", methods=["GET", "POST"])
@login_required
@admin_permission.require(http_exception=403)
def dashboard():
last_cons, now = {}, datetime.utcnow()
- users = list(UserController().read().order_by('id'))
+ users = list(UserController().read().order_by("id"))
form = InformationMessageForm()
for user in users:
last_cons[user.id] = format_timedelta(now - user.last_seen)
- return render_template('admin/dashboard.html', now=datetime.utcnow(),
- last_cons=last_cons, users=users, current_user=current_user,
- form=form)
-
-
-@admin_bp.route('/user/create', methods=['GET'])
-@admin_bp.route('/user/edit/<int:user_id>', methods=['GET'])
+ return render_template(
+ "admin/dashboard.html",
+ now=datetime.utcnow(),
+ last_cons=last_cons,
+ users=users,
+ current_user=current_user,
+ form=form,
+ )
+
+
+@admin_bp.route("/user/create", methods=["GET"])
+@admin_bp.route("/user/edit/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require(http_exception=403)
def user_form(user_id=None):
if user_id is not None:
user = UserController().get(id=user_id)
form = UserForm(obj=user)
- message = gettext('Edit the user <i>%(nick)s</i>', nick=user.nickname)
+ message = gettext("Edit the user <i>%(nick)s</i>", nick=user.nickname)
else:
form = UserForm()
- message = gettext('Add a new user')
- return render_template('/admin/create_user.html',
- form=form, message=message)
+ message = gettext("Add a new user")
+ return render_template("/admin/create_user.html", form=form, message=message)
-@admin_bp.route('/user/create', methods=['POST'])
-@admin_bp.route('/user/edit/<int:user_id>', methods=['POST'])
+@admin_bp.route("/user/create", methods=["POST"])
+@admin_bp.route("/user/edit/<int:user_id>", methods=["POST"])
@login_required
@admin_permission.require(http_exception=403)
def process_user_form(user_id=None):
@@ -53,31 +57,42 @@ def process_user_form(user_id=None):
user_contr = UserController()
if not form.validate():
- return render_template('/admin/create_user.html', form=form,
- message=gettext('Some errors were found'))
+ return render_template(
+ "/admin/create_user.html",
+ form=form,
+ message=gettext("Some errors were found"),
+ )
if user_id is not None:
# Edit a user
- user_contr.update({'id': user_id},
- {'nickname': form.nickname.data,
- 'password': form.password.data,
- 'automatic_crawling': form.automatic_crawling.data})
+ user_contr.update(
+ {"id": user_id},
+ {
+ "nickname": form.nickname.data,
+ "password": form.password.data,
+ "automatic_crawling": form.automatic_crawling.data,
+ },
+ )
user = user_contr.get(id=user_id)
- flash(gettext('User %(nick)s successfully updated',
- nick=user.nickname), 'success')
+ flash(
+ gettext("User %(nick)s successfully updated", nick=user.nickname), "success"
+ )
else:
# Create a new user (by the admin)
- user = user_contr.create(nickname=form.nickname.data,
- password=form.password.data,
- automatic_crawling=form.automatic_crawling.data,
- is_admin=False,
- is_active=True)
- flash(gettext('User %(nick)s successfully created',
- nick=user.nickname), 'success')
- return redirect(url_for('admin.user_form', user_id=user.id))
+ user = user_contr.create(
+ nickname=form.nickname.data,
+ password=form.password.data,
+ automatic_crawling=form.automatic_crawling.data,
+ is_admin=False,
+ is_active=True,
+ )
+ flash(
+ gettext("User %(nick)s successfully created", nick=user.nickname), "success"
+ )
+ return redirect(url_for("admin.user_form", user_id=user.id))
-@admin_bp.route('/delete_user/<int:user_id>', methods=['GET'])
+@admin_bp.route("/delete_user/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require(http_exception=403)
def delete_user(user_id=None):
@@ -86,16 +101,21 @@ def delete_user(user_id=None):
"""
try:
user = UserController().delete(user_id)
- flash(gettext('User %(nick)s successfully deleted',
- nick=user.nickname), 'success')
+ flash(
+ gettext("User %(nick)s successfully deleted", nick=user.nickname), "success"
+ )
except Exception as error:
flash(
- gettext('An error occurred while trying to delete a user: %(error)s',
- error=error), 'danger')
- return redirect(url_for('admin.dashboard'))
+ gettext(
+ "An error occurred while trying to delete a user: %(error)s",
+ error=error,
+ ),
+ "danger",
+ )
+ return redirect(url_for("admin.dashboard"))
-@admin_bp.route('/toggle_user/<int:user_id>', methods=['GET'])
+@admin_bp.route("/toggle_user/<int:user_id>", methods=["GET"])
@login_required
@admin_permission.require()
def toggle_user(user_id=None):
@@ -104,16 +124,18 @@ def toggle_user(user_id=None):
"""
ucontr = UserController()
user = ucontr.get(id=user_id)
- user_changed = ucontr.update({'id': user_id},
- {'is_active': not user.is_active})
+ user_changed = ucontr.update({"id": user_id}, {"is_active": not user.is_active})
if not user_changed:
- flash(gettext('This user does not exist.'), 'danger')
- return redirect(url_for('admin.dashboard'))
+ flash(gettext("This user does not exist."), "danger")
+ return redirect(url_for("admin.dashboard"))
else:
- act_txt = 'activated' if user.is_active else 'desactivated'
- message = gettext('User %(nickname)s successfully %(is_active)s',
- nickname=user.nickname, is_active=act_txt)
- flash(message, 'success')
- return redirect(url_for('admin.dashboard'))
+ act_txt = "activated" if user.is_active else "desactivated"
+ message = gettext(
+ "User %(nickname)s successfully %(is_active)s",
+ nickname=user.nickname,
+ is_active=act_txt,
+ )
+ flash(message, "success")
+ return redirect(url_for("admin.dashboard"))
diff --git a/newspipe/web/views/api/v2/__init__.py b/newspipe/web/views/api/v2/__init__.py
index 46760261..ef587e72 100644
--- a/newspipe/web/views/api/v2/__init__.py
+++ b/newspipe/web/views/api/v2/__init__.py
@@ -1,3 +1,3 @@
from web.views.api.v2 import article, feed, category
-__all__ = ['article', 'feed', 'category']
+__all__ = ["article", "feed", "category"]
diff --git a/newspipe/web/views/api/v2/article.py b/newspipe/web/views/api/v2/article.py
index 2be286c6..8da6c6dd 100644
--- a/newspipe/web/views/api/v2/article.py
+++ b/newspipe/web/views/api/v2/article.py
@@ -6,8 +6,12 @@ from flask_restful import Api
from web.views.common import api_permission
from web.controllers import ArticleController
-from web.views.api.v2.common import (PyAggAbstractResource,
- PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti)
+from web.views.api.v2.common import (
+ PyAggAbstractResource,
+ PyAggResourceNew,
+ PyAggResourceExisting,
+ PyAggResourceMulti,
+)
class ArticleNewAPI(PyAggResourceNew):
@@ -24,30 +28,32 @@ class ArticlesAPI(PyAggResourceMulti):
class ArticlesChallenge(PyAggAbstractResource):
controller_cls = ArticleController
- attrs = {'ids': {'type': list, 'default': []}}
+ attrs = {"ids": {"type": list, "default": []}}
@api_permission.require(http_exception=403)
def get(self):
- parsed_args = self.reqparse_args(right='read')
+ parsed_args = self.reqparse_args(right="read")
# collecting all attrs for casting purpose
- attrs = self.controller_cls._get_attrs_desc('admin')
- for id_dict in parsed_args['ids']:
+ attrs = self.controller_cls._get_attrs_desc("admin")
+ for id_dict in parsed_args["ids"]:
keys_to_ignore = []
for key in id_dict:
if key not in attrs:
keys_to_ignore.append(key)
- if issubclass(attrs[key]['type'], datetime):
+ if issubclass(attrs[key]["type"], datetime):
id_dict[key] = dateutil.parser.parse(id_dict[key])
for key in keys_to_ignore:
del id_dict[key]
- result = list(self.controller.challenge(parsed_args['ids']))
+ result = list(self.controller.challenge(parsed_args["ids"]))
return result or None, 200 if result else 204
+
api = Api(current_app, prefix=API_ROOT)
-api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
-api.add_resource(ArticleAPI, '/article/<int:obj_id>', endpoint='article.json')
-api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json')
-api.add_resource(ArticlesChallenge, '/articles/challenge',
- endpoint='articles_challenge.json')
+api.add_resource(ArticleNewAPI, "/article", endpoint="article_new.json")
+api.add_resource(ArticleAPI, "/article/<int:obj_id>", endpoint="article.json")
+api.add_resource(ArticlesAPI, "/articles", endpoint="articles.json")
+api.add_resource(
+ ArticlesChallenge, "/articles/challenge", endpoint="articles_challenge.json"
+)
diff --git a/newspipe/web/views/api/v2/category.py b/newspipe/web/views/api/v2/category.py
index 70fda1ea..a830624d 100644
--- a/newspipe/web/views/api/v2/category.py
+++ b/newspipe/web/views/api/v2/category.py
@@ -3,9 +3,11 @@ from flask import current_app
from flask_restful import Api
from web.controllers.category import CategoryController
-from web.views.api.v2.common import (PyAggResourceNew,
- PyAggResourceExisting,
- PyAggResourceMulti)
+from web.views.api.v2.common import (
+ PyAggResourceNew,
+ PyAggResourceExisting,
+ PyAggResourceMulti,
+)
class CategoryNewAPI(PyAggResourceNew):
@@ -21,7 +23,6 @@ class CategoriesAPI(PyAggResourceMulti):
api = Api(current_app, prefix=API_ROOT)
-api.add_resource(CategoryNewAPI, '/category', endpoint='category_new.json')
-api.add_resource(CategoryAPI, '/category/<int:obj_id>',
- endpoint='category.json')
-api.add_resource(CategoriesAPI, '/categories', endpoint='categories.json')
+api.add_resource(CategoryNewAPI, "/category", endpoint="category_new.json")
+api.add_resource(CategoryAPI, "/category/<int:obj_id>", endpoint="category.json")
+api.add_resource(CategoriesAPI, "/categories", endpoint="categories.json")
diff --git a/newspipe/web/views/api/v2/common.py b/newspipe/web/views/api/v2/common.py
index 8a53d7e6..81248422 100644
--- a/newspipe/web/views/api/v2/common.py
+++ b/newspipe/web/views/api/v2/common.py
@@ -26,8 +26,12 @@ from flask import request
from flask_restful import Resource, reqparse
from flask_login import current_user
-from web.views.common import admin_permission, api_permission, \
- login_user_bundle, jsonify
+from web.views.common import (
+ admin_permission,
+ api_permission,
+ login_user_bundle,
+ jsonify,
+)
from web.controllers import UserController
logger = logging.getLogger(__name__)
@@ -50,6 +54,7 @@ def authenticate(func):
if current_user.is_authenticated:
return func(*args, **kwargs)
raise Unauthorized()
+
return wrapper
@@ -64,8 +69,9 @@ class PyAggAbstractResource(Resource):
return self.controller_cls()
return self.controller_cls(current_user.id)
- def reqparse_args(self, right, req=None, strict=False, default=True,
- allow_empty=False):
+ def reqparse_args(
+ self, right, req=None, strict=False, default=True, allow_empty=False
+ ):
"""
strict: bool
if True will throw 400 error if args are defined and not in request
@@ -89,42 +95,41 @@ class PyAggAbstractResource(Resource):
if self.attrs is not None:
attrs = self.attrs
elif admin_permission.can():
- attrs = self.controller_cls._get_attrs_desc('admin')
+ attrs = self.controller_cls._get_attrs_desc("admin")
elif api_permission.can():
- attrs = self.controller_cls._get_attrs_desc('api', right)
+ attrs = self.controller_cls._get_attrs_desc("api", right)
else:
- attrs = self.controller_cls._get_attrs_desc('base', right)
+ attrs = self.controller_cls._get_attrs_desc("base", right)
assert attrs, "No defined attrs for %s" % self.__class__.__name__
for attr_name, attr in attrs.items():
if not default and attr_name not in in_values:
continue
else:
- parser.add_argument(attr_name, location='json',
- default=in_values[attr_name])
+ parser.add_argument(
+ attr_name, location="json", default=in_values[attr_name]
+ )
return parser.parse_args(req=request.args, strict=strict)
class PyAggResourceNew(PyAggAbstractResource):
-
@api_permission.require(http_exception=403)
def post(self):
"""Create a single new object"""
- return self.controller.create(**self.reqparse_args(right='write')), 201
+ return self.controller.create(**self.reqparse_args(right="write")), 201
class PyAggResourceExisting(PyAggAbstractResource):
-
def get(self, obj_id=None):
"""Retrieve a single object"""
return self.controller.get(id=obj_id)
def put(self, obj_id=None):
"""update an object, new attrs should be passed in the payload"""
- args = self.reqparse_args(right='write', default=False)
+ args = self.reqparse_args(right="write", default=False)
if not args:
raise BadRequest()
- return self.controller.update({'id': obj_id}, args), 200
+ return self.controller.update({"id": obj_id}, args), 200
def delete(self, obj_id=None):
"""delete a object"""
@@ -133,19 +138,18 @@ class PyAggResourceExisting(PyAggAbstractResource):
class PyAggResourceMulti(PyAggAbstractResource):
-
def get(self):
"""retrieve several objects. filters can be set in the payload on the
different fields of the object, and a limit can be set in there as well
"""
args = {}
try:
- limit = request.json.pop('limit', 10)
- order_by = request.json.pop('order_by', None)
+ limit = request.json.pop("limit", 10)
+ order_by = request.json.pop("order_by", None)
except Exception:
- args = self.reqparse_args(right='read', default=False)
- limit = request.args.get('limit', 10)
- order_by = request.args.get('order_by', None)
+ args = self.reqparse_args(right="read", default=False)
+ limit = request.args.get("limit", 10)
+ order_by = request.args.get("order_by", None)
query = self.controller.read(**args)
if order_by:
query = query.order_by(order_by)
@@ -163,10 +167,11 @@ class PyAggResourceMulti(PyAggAbstractResource):
class Proxy:
pass
+
for attrs in request.json:
try:
Proxy.json = attrs
- args = self.reqparse_args('write', req=Proxy, default=False)
+ args = self.reqparse_args("write", req=Proxy, default=False)
obj = self.controller.create(**args)
results.append(obj)
except Exception as error:
@@ -188,20 +193,21 @@ class PyAggResourceMulti(PyAggAbstractResource):
class Proxy:
pass
+
for obj_id, attrs in request.json:
try:
Proxy.json = attrs
- args = self.reqparse_args('write', req=Proxy, default=False)
- result = self.controller.update({'id': obj_id}, args)
+ args = self.reqparse_args("write", req=Proxy, default=False)
+ result = self.controller.update({"id": obj_id}, args)
if result:
- results.append('ok')
+ results.append("ok")
else:
- results.append('nok')
+ results.append("nok")
except Exception as error:
results.append(str(error))
- if results.count('ok') == 0: # all failed => 500
+ if results.count("ok") == 0: # all failed => 500
status = 500
- elif results.count('ok') != len(results): # some failed => 206
+ elif results.count("ok") != len(results): # some failed => 206
status = 206
return results, status
@@ -212,11 +218,11 @@ class PyAggResourceMulti(PyAggAbstractResource):
for obj_id in request.json:
try:
self.controller.delete(obj_id)
- results.append('ok')
+ results.append("ok")
except Exception as error:
status = 206
results.append(error)
# if no operation succeeded, it's not partial anymore, returning err 500
- if status == 206 and results.count('ok') == 0:
+ if status == 206 and results.count("ok") == 0:
status = 500
return results, status
diff --git a/newspipe/web/views/api/v2/feed.py b/newspipe/web/views/api/v2/feed.py
index a0691277..1e4fabf2 100644
--- a/newspipe/web/views/api/v2/feed.py
+++ b/newspipe/web/views/api/v2/feed.py
@@ -3,14 +3,14 @@ from flask import current_app
from flask_restful import Api
from web.views.common import api_permission
-from web.controllers.feed import (FeedController,
- DEFAULT_MAX_ERROR,
- DEFAULT_LIMIT)
+from web.controllers.feed import FeedController, DEFAULT_MAX_ERROR, DEFAULT_LIMIT
-from web.views.api.v2.common import PyAggAbstractResource, \
- PyAggResourceNew, \
- PyAggResourceExisting, \
- PyAggResourceMulti
+from web.views.api.v2.common import (
+ PyAggAbstractResource,
+ PyAggResourceNew,
+ PyAggResourceExisting,
+ PyAggResourceMulti,
+)
class FeedNewAPI(PyAggResourceNew):
@@ -27,21 +27,21 @@ class FeedsAPI(PyAggResourceMulti):
class FetchableFeedAPI(PyAggAbstractResource):
controller_cls = FeedController
- attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR},
- 'limit': {'type': int, 'default': DEFAULT_LIMIT}}
+ attrs = {
+ "max_error": {"type": int, "default": DEFAULT_MAX_ERROR},
+ "limit": {"type": int, "default": DEFAULT_LIMIT},
+ }
@api_permission.require(http_exception=403)
def get(self):
- args = self.reqparse_args(right='read', allow_empty=True)
- result = [feed for feed
- in self.controller.list_fetchable(**args)]
+ args = self.reqparse_args(right="read", allow_empty=True)
+ result = [feed for feed in self.controller.list_fetchable(**args)]
return result or None, 200 if result else 204
api = Api(current_app, prefix=API_ROOT)
-api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')
-api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json')
-api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json')
-api.add_resource(FetchableFeedAPI, '/feeds/fetchable',
- endpoint='fetchable_feed.json')
+api.add_resource(FeedNewAPI, "/feed", endpoint="feed_new.json")
+api.add_resource(FeedAPI, "/feed/<int:obj_id>", endpoint="feed.json")
+api.add_resource(FeedsAPI, "/feeds", endpoint="feeds.json")
+api.add_resource(FetchableFeedAPI, "/feeds/fetchable", endpoint="fetchable_feed.json")
diff --git a/newspipe/web/views/api/v3/__init__.py b/newspipe/web/views/api/v3/__init__.py
index 76aa1f19..5066b0b6 100644
--- a/newspipe/web/views/api/v3/__init__.py
+++ b/newspipe/web/views/api/v3/__init__.py
@@ -1,3 +1,3 @@
from web.views.api.v3 import article
-__all__ = ['article']
+__all__ = ["article"]
diff --git a/newspipe/web/views/api/v3/article.py b/newspipe/web/views/api/v3/article.py
index 4cf35648..04d19552 100644
--- a/newspipe/web/views/api/v3/article.py
+++ b/newspipe/web/views/api/v3/article.py
@@ -35,6 +35,7 @@ from web.controllers import ArticleController, FeedController
from web.views.api.v3.common import AbstractProcessor
from web.views.api.v3.common import url_prefix, auth_func
+
class ArticleProcessor(AbstractProcessor):
"""Concrete processors for the Article Web service.
"""
@@ -43,7 +44,7 @@ class ArticleProcessor(AbstractProcessor):
try:
article = ArticleController(current_user.id).get(id=instance_id)
except NotFound:
- raise ProcessingException(description='No such article.', code=404)
+ raise ProcessingException(description="No such article.", code=404)
self.is_authorized(current_user, article)
def post_preprocessor(self, data=None, **kw):
@@ -52,7 +53,7 @@ class ArticleProcessor(AbstractProcessor):
try:
feed = FeedController(current_user.id).get(id=data["feed_id"])
except NotFound:
- raise ProcessingException(description='No such feed.', code=404)
+ raise ProcessingException(description="No such feed.", code=404)
self.is_authorized(current_user, feed)
data["category_id"] = feed.category_id
@@ -61,24 +62,23 @@ class ArticleProcessor(AbstractProcessor):
try:
article = ArticleController(current_user.id).get(id=instance_id)
except NotFound:
- raise ProcessingException(description='No such article.', code=404)
+ raise ProcessingException(description="No such article.", code=404)
self.is_authorized(current_user, article)
+
article_processor = ArticleProcessor()
-blueprint_article = manager.create_api_blueprint(models.Article,
- url_prefix=url_prefix,
- methods=['GET', 'POST', 'PUT', 'DELETE'],
- preprocessors=dict(GET_SINGLE=[auth_func,
- article_processor.get_single_preprocessor],
- GET_MANY=[auth_func,
- article_processor.get_many_preprocessor],
- POST=[auth_func,
- article_processor.post_preprocessor],
- PUT_SINGLE=[auth_func,
- article_processor.put_single_preprocessor],
- PUT_MANY=[auth_func,
- article_processor.put_many_preprocessor],
- DELETE=[auth_func,
- article_processor.delete_preprocessor]))
+blueprint_article = manager.create_api_blueprint(
+ models.Article,
+ url_prefix=url_prefix,
+ methods=["GET", "POST", "PUT", "DELETE"],
+ preprocessors=dict(
+ GET_SINGLE=[auth_func, article_processor.get_single_preprocessor],
+ GET_MANY=[auth_func, article_processor.get_many_preprocessor],
+ POST=[auth_func, article_processor.post_preprocessor],
+ PUT_SINGLE=[auth_func, article_processor.put_single_preprocessor],
+ PUT_MANY=[auth_func, article_processor.put_many_preprocessor],
+ DELETE=[auth_func, article_processor.delete_preprocessor],
+ ),
+)
application.register_blueprint(blueprint_article)
diff --git a/newspipe/web/views/api/v3/common.py b/newspipe/web/views/api/v3/common.py
index d5e94a3f..5467ee30 100644
--- a/newspipe/web/views/api/v3/common.py
+++ b/newspipe/web/views/api/v3/common.py
@@ -33,7 +33,8 @@ from werkzeug.exceptions import NotFound
from web.controllers import ArticleController, UserController
from web.views.common import login_user_bundle
-url_prefix = '/api/v3'
+url_prefix = "/api/v3"
+
def auth_func(*args, **kw):
if request.authorization:
@@ -41,24 +42,23 @@ def auth_func(*args, **kw):
try:
user = ucontr.get(nickname=request.authorization.username)
except NotFound:
- raise ProcessingException("Couldn't authenticate your user",
- code=401)
+ raise ProcessingException("Couldn't authenticate your user", code=401)
if not ucontr.check_password(user, request.authorization.password):
- raise ProcessingException("Couldn't authenticate your user",
- code=401)
+ raise ProcessingException("Couldn't authenticate your user", code=401)
if not user.is_active:
raise ProcessingException("User is deactivated", code=401)
login_user_bundle(user)
if not current_user.is_authenticated:
- raise ProcessingException(description='Not authenticated!', code=401)
+ raise ProcessingException(description="Not authenticated!", code=401)
+
-class AbstractProcessor():
+class AbstractProcessor:
"""Abstract processors for the Web services.
"""
def is_authorized(self, user, obj):
if user.id != obj.user_id:
- raise ProcessingException(description='Not Authorized', code=401)
+ raise ProcessingException(description="Not Authorized", code=401)
def get_single_preprocessor(self, instance_id=None, **kw):
# Check if the user is authorized to modify the specified
@@ -69,13 +69,11 @@ class AbstractProcessor():
"""Accepts a single argument, `search_params`, which is a dictionary
containing the search parameters for the request.
"""
- filt = dict(name="user_id",
- op="eq",
- val=current_user.id)
+ filt = dict(name="user_id", op="eq", val=current_user.id)
# Check if there are any filters there already.
if "filters" not in search_params:
- search_params["filters"] = []
+ search_params["filters"] = []
search_params["filters"].append(filt)
@@ -95,13 +93,11 @@ class AbstractProcessor():
is a dictionary representing the fields to change on the matching
instances and the values to which they will be set.
"""
- filt = dict(name="user_id",
- op="eq",
- val=current_user.id)
+ filt = dict(name="user_id", op="eq", val=current_user.id)
# Check if there are any filters there already.
if "filters" not in search_params:
- search_params["filters"] = []
+ search_params["filters"] = []
search_params["filters"].append(filt)
diff --git a/newspipe/web/views/api/v3/feed.py b/newspipe/web/views/api/v3/feed.py
index 2cbbafd9..9d2c9180 100644
--- a/newspipe/web/views/api/v3/feed.py
+++ b/newspipe/web/views/api/v3/feed.py
@@ -33,6 +33,7 @@ from web.controllers import FeedController
from web.views.api.v3.common import AbstractProcessor
from web.views.api.v3.common import url_prefix, auth_func
+
class FeedProcessor(AbstractProcessor):
"""Concrete processors for the Feed Web service.
"""
@@ -43,16 +44,19 @@ class FeedProcessor(AbstractProcessor):
feed = FeedController(current_user.id).get(id=instance_id)
self.is_authorized(current_user, feed)
+
feed_processor = FeedProcessor()
-blueprint_feed = manager.create_api_blueprint(models.Feed,
- url_prefix=url_prefix,
- methods=['GET', 'POST', 'PUT', 'DELETE'],
- preprocessors=dict(GET_SINGLE=[auth_func,
- feed_processor.get_single_preprocessor],
- GET_MANY=[auth_func,
- feed_processor.get_many_preprocessor],
- PUT_SINGLE=[auth_func],
- POST=[auth_func],
- DELETE=[auth_func]))
+blueprint_feed = manager.create_api_blueprint(
+ models.Feed,
+ url_prefix=url_prefix,
+ methods=["GET", "POST", "PUT", "DELETE"],
+ preprocessors=dict(
+ GET_SINGLE=[auth_func, feed_processor.get_single_preprocessor],
+ GET_MANY=[auth_func, feed_processor.get_many_preprocessor],
+ PUT_SINGLE=[auth_func],
+ POST=[auth_func],
+ DELETE=[auth_func],
+ ),
+)
application.register_blueprint(blueprint_feed)
diff --git a/newspipe/web/views/article.py b/newspipe/web/views/article.py
index bf39795d..ba66726e 100644
--- a/newspipe/web/views/article.py
+++ b/newspipe/web/views/article.py
@@ -1,6 +1,14 @@
from datetime import datetime, timedelta
-from flask import (Blueprint, g, render_template, redirect,
- flash, url_for, make_response, request)
+from flask import (
+ Blueprint,
+ g,
+ render_template,
+ redirect,
+ flash,
+ url_for,
+ make_response,
+ request,
+)
from flask_babel import gettext
from flask_login import login_required, current_user
@@ -9,25 +17,24 @@ from flask_login import login_required, current_user
from bootstrap import db
from lib.utils import clear_string, redirect_url
from lib.data import export_json
-from web.controllers import (ArticleController, UserController,
- CategoryController)
+from web.controllers import ArticleController, UserController, CategoryController
from web.lib.view_utils import etag_match
-articles_bp = Blueprint('articles', __name__, url_prefix='/articles')
-article_bp = Blueprint('article', __name__, url_prefix='/article')
+articles_bp = Blueprint("articles", __name__, url_prefix="/articles")
+article_bp = Blueprint("article", __name__, url_prefix="/article")
-@article_bp.route('/redirect/<int:article_id>', methods=['GET'])
+@article_bp.route("/redirect/<int:article_id>", methods=["GET"])
@login_required
def redirect_to_article(article_id):
contr = ArticleController(current_user.id)
article = contr.get(id=article_id)
if not article.readed:
- contr.update({'id': article.id}, {'readed': True})
+ contr.update({"id": article.id}, {"readed": True})
return redirect(article.link)
-@article_bp.route('/<int:article_id>', methods=['GET'])
+@article_bp.route("/<int:article_id>", methods=["GET"])
@login_required
@etag_match
def article(article_id=None):
@@ -35,11 +42,12 @@ def article(article_id=None):
Presents an article.
"""
article = ArticleController(current_user.id).get(id=article_id)
- return render_template('article.html',
- head_titles=[clear_string(article.title)],
- article=article)
+ return render_template(
+ "article.html", head_titles=[clear_string(article.title)], article=article
+ )
-@article_bp.route('/public/<int:article_id>', methods=['GET'])
+
+@article_bp.route("/public/<int:article_id>", methods=["GET"])
@etag_match
def article_pub(article_id=None):
"""
@@ -48,13 +56,13 @@ def article_pub(article_id=None):
"""
article = ArticleController().get(id=article_id)
if article.source.private or not article.source.user.is_public_profile:
- return render_template('errors/404.html'), 404
- return render_template('article_pub.html',
- head_titles=[clear_string(article.title)],
- article=article)
+ return render_template("errors/404.html"), 404
+ return render_template(
+ "article_pub.html", head_titles=[clear_string(article.title)], article=article
+ )
-@article_bp.route('/like/<int:article_id>', methods=['GET'])
+@article_bp.route("/like/<int:article_id>", methods=["GET"])
@login_required
def like(article_id=None):
"""
@@ -62,80 +70,84 @@ def like(article_id=None):
"""
art_contr = ArticleController(current_user.id)
article = art_contr.get(id=article_id)
- art_contr = art_contr.update({'id': article_id},
- {'like': not article.like})
+ art_contr = art_contr.update({"id": article_id}, {"like": not article.like})
return redirect(redirect_url())
-@article_bp.route('/delete/<int:article_id>', methods=['GET'])
+@article_bp.route("/delete/<int:article_id>", methods=["GET"])
@login_required
def delete(article_id=None):
"""
Delete an article from the database.
"""
article = ArticleController(current_user.id).delete(article_id)
- flash(gettext('Article %(article_title)s deleted',
- article_title=article.title), 'success')
- return redirect(url_for('home'))
+ flash(
+ gettext("Article %(article_title)s deleted", article_title=article.title),
+ "success",
+ )
+ return redirect(url_for("home"))
-@articles_bp.route('/history', methods=['GET'])
-@articles_bp.route('/history/<int:year>', methods=['GET'])
-@articles_bp.route('/history/<int:year>/<int:month>', methods=['GET'])
+@articles_bp.route("/history", methods=["GET"])
+@articles_bp.route("/history/<int:year>", methods=["GET"])
+@articles_bp.route("/history/<int:year>/<int:month>", methods=["GET"])
@login_required
def history(year=None, month=None):
cntr, artcles = ArticleController(current_user.id).get_history(year, month)
- return render_template('history.html', articles_counter=cntr,
- articles=artcles, year=year, month=month)
+ return render_template(
+ "history.html", articles_counter=cntr, articles=artcles, year=year, month=month
+ )
-@article_bp.route('/mark_as/<string:new_value>', methods=['GET'])
-@article_bp.route('/mark_as/<string:new_value>/article/<int:article_id>',
- methods=['GET'])
+@article_bp.route("/mark_as/<string:new_value>", methods=["GET"])
+@article_bp.route(
+ "/mark_as/<string:new_value>/article/<int:article_id>", methods=["GET"]
+)
@login_required
-def mark_as(new_value='read', feed_id=None, article_id=None):
+def mark_as(new_value="read", feed_id=None, article_id=None):
"""
Mark all unreaded articles as read.
"""
- readed = new_value == 'read'
+ readed = new_value == "read"
art_contr = ArticleController(current_user.id)
- filters = {'readed': not readed}
+ filters = {"readed": not readed}
if feed_id is not None:
- filters['feed_id'] = feed_id
- message = 'Feed marked as %s.'
+ filters["feed_id"] = feed_id
+ message = "Feed marked as %s."
elif article_id is not None:
- filters['id'] = article_id
- message = 'Article marked as %s.'
+ filters["id"] = article_id
+ message = "Article marked as %s."
else:
- message = 'All article marked as %s.'
+ message = "All article marked as %s."
art_contr.update(filters, {"readed": readed})
- flash(gettext(message % new_value), 'info')
+ flash(gettext(message % new_value), "info")
if readed:
return redirect(redirect_url())
- return redirect('home')
+ return redirect("home")
-@articles_bp.route('/expire_articles', methods=['GET'])
+@articles_bp.route("/expire_articles", methods=["GET"])
@login_required
def expire():
"""
Delete articles older than the given number of weeks.
"""
current_time = datetime.utcnow()
- weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10)))
+ weeks_ago = current_time - timedelta(int(request.args.get("weeks", 10)))
art_contr = ArticleController(current_user.id)
- query = art_contr.read(__or__={'date__lt': weeks_ago,
- 'retrieved_date__lt': weeks_ago})
+ query = art_contr.read(
+ __or__={"date__lt": weeks_ago, "retrieved_date__lt": weeks_ago}
+ )
count = query.count()
query.delete()
db.session.commit()
- flash(gettext('%(count)d articles deleted', count=count), 'info')
+ flash(gettext("%(count)d articles deleted", count=count), "info")
return redirect(redirect_url())
-@articles_bp.route('/export', methods=['GET'])
+@articles_bp.route("/export", methods=["GET"])
@login_required
def export():
"""
@@ -145,10 +157,9 @@ def export():
try:
json_result = export_json(user)
except Exception as e:
- flash(gettext("Error when exporting articles."), 'danger')
+ flash(gettext("Error when exporting articles."), "danger")
return redirect(redirect_url())
response = make_response(json_result)
- response.mimetype = 'application/json'
- response.headers["Content-Disposition"] \
- = 'attachment; filename=account.json'
+ response.mimetype = "application/json"
+ response.headers["Content-Disposition"] = "attachment; filename=account.json"
return response
diff --git a/newspipe/web/views/bookmark.py b/newspipe/web/views/bookmark.py
index 21d832d2..bceda631 100644
--- a/newspipe/web/views/bookmark.py
+++ b/newspipe/web/views/bookmark.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
# Newspipe - A Web based news aggregator.
# Copyright (C) 2010-2017 Cédric Bonhomme - https://www.cedricbonhomme.org
@@ -30,8 +30,15 @@ import logging
import datetime
from werkzeug.exceptions import BadRequest
-from flask import Blueprint, render_template, flash, \
- redirect, request, url_for, make_response
+from flask import (
+ Blueprint,
+ render_template,
+ flash,
+ redirect,
+ request,
+ url_for,
+ make_response,
+)
from flask_babel import gettext
from flask_login import login_required, current_user
from flask_paginate import Pagination, get_page_args
@@ -46,93 +53,105 @@ from web.controllers import BookmarkController, BookmarkTagController
from web.models import BookmarkTag
logger = logging.getLogger(__name__)
-bookmarks_bp = Blueprint('bookmarks', __name__, url_prefix='/bookmarks')
-bookmark_bp = Blueprint('bookmark', __name__, url_prefix='/bookmark')
+bookmarks_bp = Blueprint("bookmarks", __name__, url_prefix="/bookmarks")
+bookmark_bp = Blueprint("bookmark", __name__, url_prefix="/bookmark")
-@bookmarks_bp.route('/', defaults={'per_page': '50'}, methods=['GET'])
-@bookmarks_bp.route('/<string:status>', defaults={'per_page': '50'},
- methods=['GET'])
-def list_(per_page, status='all'):
+@bookmarks_bp.route("/", defaults={"per_page": "50"}, methods=["GET"])
+@bookmarks_bp.route("/<string:status>", defaults={"per_page": "50"}, methods=["GET"])
+def list_(per_page, status="all"):
"Lists the bookmarks."
head_titles = [gettext("Bookmarks")]
user_id = None
filters = {}
- tag = request.args.get('tag', None)
+ tag = request.args.get("tag", None)
if tag:
- filters['tags_proxy__contains'] = tag
- query = request.args.get('query', None)
+ filters["tags_proxy__contains"] = tag
+ query = request.args.get("query", None)
if query:
- query_regex = '%' + query + '%'
- filters['__or__'] = {'title__ilike': query_regex,
- 'description__ilike': query_regex}
+ query_regex = "%" + query + "%"
+ filters["__or__"] = {
+ "title__ilike": query_regex,
+ "description__ilike": query_regex,
+ }
if current_user.is_authenticated:
# query for the bookmarks of the authenticated user
user_id = current_user.id
- if status == 'public':
+ if status == "public":
# load public bookmarks only
- filters['shared'] = True
- elif status == 'private':
+ filters["shared"] = True
+ elif status == "private":
# load private bookmarks only
- filters['shared'] = False
+ filters["shared"] = False
else:
# no filter: load shared and public bookmarks
pass
- if status == 'unread':
- filters['to_read'] = True
+ if status == "unread":
+ filters["to_read"] = True
else:
pass
else:
# query for the shared bookmarks (of all users)
head_titles = [gettext("Recent bookmarks")]
- not_created_before = datetime.datetime.today() - \
- datetime.timedelta(days=900)
- filters['time__gt'] = not_created_before # only "recent" bookmarks
- filters['shared'] = True
+ not_created_before = datetime.datetime.today() - datetime.timedelta(days=900)
+ filters["time__gt"] = not_created_before # only "recent" bookmarks
+ filters["shared"] = True
- bookmarks = BookmarkController(user_id) \
- .read(**filters) \
- .order_by(desc('time'))
+ bookmarks = BookmarkController(user_id).read(**filters).order_by(desc("time"))
- #tag_contr = BookmarkTagController(user_id)
- #tag_contr.read().join(bookmarks).all()
+ # tag_contr = BookmarkTagController(user_id)
+ # tag_contr.read().join(bookmarks).all()
page, per_page, offset = get_page_args()
- pagination = Pagination(page=page, total=bookmarks.count(),
- css_framework='bootstrap3',
- search=False, record_name='bookmarks',
- per_page=per_page)
-
- return render_template('bookmarks.html',
- head_titles=head_titles,
- bookmarks=bookmarks.offset(offset).limit(per_page),
- pagination=pagination,
- tag=tag,
- query=query)
-
-
-@bookmark_bp.route('/create', methods=['GET'])
-@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['GET'])
+ pagination = Pagination(
+ page=page,
+ total=bookmarks.count(),
+ css_framework="bootstrap3",
+ search=False,
+ record_name="bookmarks",
+ per_page=per_page,
+ )
+
+ return render_template(
+ "bookmarks.html",
+ head_titles=head_titles,
+ bookmarks=bookmarks.offset(offset).limit(per_page),
+ pagination=pagination,
+ tag=tag,
+ query=query,
+ )
+
+
+@bookmark_bp.route("/create", methods=["GET"])
+@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["GET"])
@login_required
def form(bookmark_id=None):
"Form to create/edit bookmarks."
action = gettext("Add a new bookmark")
head_titles = [action]
if bookmark_id is None:
- return render_template('edit_bookmark.html', action=action,
- head_titles=head_titles, form=BookmarkForm())
+ return render_template(
+ "edit_bookmark.html",
+ action=action,
+ head_titles=head_titles,
+ form=BookmarkForm(),
+ )
bookmark = BookmarkController(current_user.id).get(id=bookmark_id)
- action = gettext('Edit bookmark')
+ action = gettext("Edit bookmark")
head_titles = [action]
form = BookmarkForm(obj=bookmark)
form.tags.data = ", ".join(bookmark.tags_proxy)
- return render_template('edit_bookmark.html', action=action,
- head_titles=head_titles, bookmark=bookmark,
- form=form)
+ return render_template(
+ "edit_bookmark.html",
+ action=action,
+ head_titles=head_titles,
+ bookmark=bookmark,
+ form=form,
+ )
-@bookmark_bp.route('/create', methods=['POST'])
-@bookmark_bp.route('/edit/<int:bookmark_id>', methods=['POST'])
+@bookmark_bp.route("/create", methods=["POST"])
+@bookmark_bp.route("/edit/<int:bookmark_id>", methods=["POST"])
@login_required
def process_form(bookmark_id=None):
"Process the creation/edition of bookmarks."
@@ -141,116 +160,131 @@ def process_form(bookmark_id=None):
tag_contr = BookmarkTagController(current_user.id)
if not form.validate():
- return render_template('edit_bookmark.html', form=form)
+ return render_template("edit_bookmark.html", form=form)
- if form.title.data == '':
+ if form.title.data == "":
title = form.href.data
else:
title = form.title.data
- bookmark_attr = {'href': form.href.data,
- 'description': form.description.data,
- 'title': title,
- 'shared': form.shared.data,
- 'to_read': form.to_read.data}
+ bookmark_attr = {
+ "href": form.href.data,
+ "description": form.description.data,
+ "title": title,
+ "shared": form.shared.data,
+ "to_read": form.to_read.data,
+ }
if bookmark_id is not None:
tags = []
- for tag in form.tags.data.split(','):
- new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id,
- bookmark_id=bookmark_id)
+ for tag in form.tags.data.split(","):
+ new_tag = tag_contr.create(
+ text=tag.strip(), user_id=current_user.id, bookmark_id=bookmark_id
+ )
tags.append(new_tag)
- bookmark_attr['tags'] = tags
- bookmark_contr.update({'id': bookmark_id}, bookmark_attr)
- flash(gettext('Bookmark successfully updated.'), 'success')
- return redirect(url_for('bookmark.form', bookmark_id=bookmark_id))
+ bookmark_attr["tags"] = tags
+ bookmark_contr.update({"id": bookmark_id}, bookmark_attr)
+ flash(gettext("Bookmark successfully updated."), "success")
+ return redirect(url_for("bookmark.form", bookmark_id=bookmark_id))
# Create a new bookmark
new_bookmark = bookmark_contr.create(**bookmark_attr)
tags = []
- for tag in form.tags.data.split(','):
- new_tag = tag_contr.create(text=tag.strip(), user_id=current_user.id,
- bookmark_id=new_bookmark.id)
+ for tag in form.tags.data.split(","):
+ new_tag = tag_contr.create(
+ text=tag.strip(), user_id=current_user.id, bookmark_id=new_bookmark.id
+ )
tags.append(new_tag)
- bookmark_attr['tags'] = tags
- bookmark_contr.update({'id': new_bookmark.id}, bookmark_attr)
- flash(gettext('Bookmark successfully created.'), 'success')
- return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id))
+ bookmark_attr["tags"] = tags
+ bookmark_contr.update({"id": new_bookmark.id}, bookmark_attr)
+ flash(gettext("Bookmark successfully created."), "success")
+ return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id))
-@bookmark_bp.route('/delete/<int:bookmark_id>', methods=['GET'])
+@bookmark_bp.route("/delete/<int:bookmark_id>", methods=["GET"])
@login_required
def delete(bookmark_id=None):
"Delete a bookmark."
bookmark = BookmarkController(current_user.id).delete(bookmark_id)
- flash(gettext("Bookmark %(bookmark_name)s successfully deleted.",
- bookmark_name=bookmark.title), 'success')
- return redirect(url_for('bookmarks.list_'))
+ flash(
+ gettext(
+ "Bookmark %(bookmark_name)s successfully deleted.",
+ bookmark_name=bookmark.title,
+ ),
+ "success",
+ )
+ return redirect(url_for("bookmarks.list_"))
-@bookmarks_bp.route('/delete', methods=['GET'])
+@bookmarks_bp.route("/delete", methods=["GET"])
@login_required
def delete_all():
"Delete all bookmarks."
bookmark = BookmarkController(current_user.id).read().delete()
db.session.commit()
- flash(gettext("Bookmarks successfully deleted."), 'success')
+ flash(gettext("Bookmarks successfully deleted."), "success")
return redirect(redirect_url())
-@bookmark_bp.route('/bookmarklet', methods=['GET', 'POST'])
+@bookmark_bp.route("/bookmarklet", methods=["GET", "POST"])
@login_required
def bookmarklet():
bookmark_contr = BookmarkController(current_user.id)
- href = (request.args if request.method == 'GET' else request.form)\
- .get('href', None)
+ href = (request.args if request.method == "GET" else request.form).get("href", None)
if not href:
flash(gettext("Couldn't add bookmark: url missing."), "error")
raise BadRequest("url is missing")
- title = (request.args if request.method == 'GET' else request.form)\
- .get('title', None)
+ title = (request.args if request.method == "GET" else request.form).get(
+ "title", None
+ )
if not title:
title = href
- bookmark_exists = bookmark_contr.read(**{'href': href}).all()
+ bookmark_exists = bookmark_contr.read(**{"href": href}).all()
if bookmark_exists:
- flash(gettext("Couldn't add bookmark: bookmark already exists."),
- "warning")
- return redirect(url_for('bookmark.form',
- bookmark_id=bookmark_exists[0].id))
+ flash(gettext("Couldn't add bookmark: bookmark already exists."), "warning")
+ return redirect(url_for("bookmark.form", bookmark_id=bookmark_exists[0].id))
- bookmark_attr = {'href': href,
- 'description': '',
- 'title': title,
- 'shared': True,
- 'to_read': True}
+ bookmark_attr = {
+ "href": href,
+ "description": "",
+ "title": title,
+ "shared": True,
+ "to_read": True,
+ }
new_bookmark = bookmark_contr.create(**bookmark_attr)
- flash(gettext('Bookmark successfully created.'), 'success')
- return redirect(url_for('bookmark.form', bookmark_id=new_bookmark.id))
+ flash(gettext("Bookmark successfully created."), "success")
+ return redirect(url_for("bookmark.form", bookmark_id=new_bookmark.id))
-@bookmark_bp.route('/import_pinboard', methods=['POST'])
+@bookmark_bp.route("/import_pinboard", methods=["POST"])
@login_required
def import_pinboard():
- bookmarks = request.files.get('jsonfile', None)
+ bookmarks = request.files.get("jsonfile", None)
if bookmarks:
try:
nb_bookmarks = import_pinboard_json(current_user, bookmarks.read())
- flash(gettext("%(nb_bookmarks)s bookmarks successfully imported.",
- nb_bookmarks=nb_bookmarks), 'success')
+ flash(
+ gettext(
+ "%(nb_bookmarks)s bookmarks successfully imported.",
+ nb_bookmarks=nb_bookmarks,
+ ),
+ "success",
+ )
except Exception as e:
- flash(gettext('Error when importing bookmarks.'), 'error')
+ flash(gettext("Error when importing bookmarks."), "error")
return redirect(redirect_url())
-@bookmarks_bp.route('/export', methods=['GET'])
+@bookmarks_bp.route("/export", methods=["GET"])
@login_required
def export():
bookmarks = export_bookmarks(current_user)
response = make_response(bookmarks)
- response.mimetype = 'application/json'
- response.headers["Content-Disposition"] \
- = 'attachment; filename=newspipe_bookmarks_export.json'
+ response.mimetype = "application/json"
+ response.headers[
+ "Content-Disposition"
+ ] = "attachment; filename=newspipe_bookmarks_export.json"
return response
diff --git a/newspipe/web/views/category.py b/newspipe/web/views/category.py
index 138561dd..1c897058 100644
--- a/newspipe/web/views/category.py
+++ b/newspipe/web/views/category.py
@@ -5,82 +5,105 @@ from flask_login import login_required, current_user
from web.forms import CategoryForm
from lib.utils import redirect_url
from web.lib.view_utils import etag_match
-from web.controllers import ArticleController, FeedController, \
- CategoryController
+from web.controllers import ArticleController, FeedController, CategoryController
-categories_bp = Blueprint('categories', __name__, url_prefix='/categories')
-category_bp = Blueprint('category', __name__, url_prefix='/category')
+categories_bp = Blueprint("categories", __name__, url_prefix="/categories")
+category_bp = Blueprint("category", __name__, url_prefix="/category")
-@categories_bp.route('/', methods=['GET'])
+@categories_bp.route("/", methods=["GET"])
@login_required
@etag_match
def list_():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
- return render_template('categories.html',
- categories=list(CategoryController(current_user.id).read().order_by('name')),
- feeds_count=FeedController(current_user.id).count_by_category(),
- unread_article_count=art_contr.count_by_category(readed=False),
- article_count=art_contr.count_by_category())
+ return render_template(
+ "categories.html",
+ categories=list(CategoryController(current_user.id).read().order_by("name")),
+ feeds_count=FeedController(current_user.id).count_by_category(),
+ unread_article_count=art_contr.count_by_category(readed=False),
+ article_count=art_contr.count_by_category(),
+ )
-@category_bp.route('/create', methods=['GET'])
-@category_bp.route('/edit/<int:category_id>', methods=['GET'])
+@category_bp.route("/create", methods=["GET"])
+@category_bp.route("/edit/<int:category_id>", methods=["GET"])
@login_required
@etag_match
def form(category_id=None):
action = gettext("Add a category")
head_titles = [action]
if category_id is None:
- return render_template('edit_category.html', action=action,
- head_titles=head_titles, form=CategoryForm())
+ return render_template(
+ "edit_category.html",
+ action=action,
+ head_titles=head_titles,
+ form=CategoryForm(),
+ )
category = CategoryController(current_user.id).get(id=category_id)
- action = gettext('Edit category')
+ action = gettext("Edit category")
head_titles = [action]
if category.name:
head_titles.append(category.name)
- return render_template('edit_category.html', action=action,
- head_titles=head_titles, category=category,
- form=CategoryForm(obj=category))
+ return render_template(
+ "edit_category.html",
+ action=action,
+ head_titles=head_titles,
+ category=category,
+ form=CategoryForm(obj=category),
+ )
-@category_bp.route('/delete/<int:category_id>', methods=['GET'])
+@category_bp.route("/delete/<int:category_id>", methods=["GET"])
@login_required
def delete(category_id=None):
category = CategoryController(current_user.id).delete(category_id)
- flash(gettext("Category %(category_name)s successfully deleted.",
- category_name=category.name), 'success')
+ flash(
+ gettext(
+ "Category %(category_name)s successfully deleted.",
+ category_name=category.name,
+ ),
+ "success",
+ )
return redirect(redirect_url())
-@category_bp.route('/create', methods=['POST'])
-@category_bp.route('/edit/<int:category_id>', methods=['POST'])
+@category_bp.route("/create", methods=["POST"])
+@category_bp.route("/edit/<int:category_id>", methods=["POST"])
@login_required
def process_form(category_id=None):
form = CategoryForm()
cat_contr = CategoryController(current_user.id)
if not form.validate():
- return render_template('edit_category.html', form=form)
+ return render_template("edit_category.html", form=form)
existing_cats = list(cat_contr.read(name=form.name.data))
if existing_cats and category_id is None:
flash(gettext("Couldn't add category: already exists."), "warning")
- return redirect(url_for('category.form',
- category_id=existing_cats[0].id))
+ return redirect(url_for("category.form", category_id=existing_cats[0].id))
# Edit an existing category
- category_attr = {'name': form.name.data}
+ category_attr = {"name": form.name.data}
if category_id is not None:
- cat_contr.update({'id': category_id}, category_attr)
- flash(gettext('Category %(cat_name)r successfully updated.',
- cat_name=category_attr['name']), 'success')
- return redirect(url_for('category.form', category_id=category_id))
+ cat_contr.update({"id": category_id}, category_attr)
+ flash(
+ gettext(
+ "Category %(cat_name)r successfully updated.",
+ cat_name=category_attr["name"],
+ ),
+ "success",
+ )
+ return redirect(url_for("category.form", category_id=category_id))
# Create a new category
new_category = cat_contr.create(**category_attr)
- flash(gettext('Category %(category_name)r successfully created.',
- category_name=new_category.name), 'success')
+ flash(
+ gettext(
+ "Category %(category_name)r successfully created.",
+ category_name=new_category.name,
+ ),
+ "success",
+ )
- return redirect(url_for('category.form', category_id=new_category.id))
+ return redirect(url_for("category.form", category_id=new_category.id))
diff --git a/newspipe/web/views/common.py b/newspipe/web/views/common.py
index e422fd57..c2d8e2df 100644
--- a/newspipe/web/views/common.py
+++ b/newspipe/web/views/common.py
@@ -3,13 +3,18 @@ from functools import wraps
from datetime import datetime
from flask import current_app, Response
from flask_login import login_user
-from flask_principal import (Identity, Permission, RoleNeed,
- session_identity_loader, identity_changed)
+from flask_principal import (
+ Identity,
+ Permission,
+ RoleNeed,
+ session_identity_loader,
+ identity_changed,
+)
from web.controllers import UserController
from lib.utils import default_handler
-admin_role = RoleNeed('admin')
-api_role = RoleNeed('api')
+admin_role = RoleNeed("admin")
+api_role = RoleNeed("api")
admin_permission = Permission(admin_role)
api_permission = Permission(api_role)
@@ -17,21 +22,23 @@ api_permission = Permission(api_role)
def scoped_default_handler():
if admin_permission.can():
- role = 'admin'
+ role = "admin"
elif api_permission.can():
- role = 'api'
+ role = "api"
else:
- role = 'user'
+ role = "user"
@wraps(default_handler)
def wrapper(obj):
return default_handler(obj, role=role)
+
return wrapper
def jsonify(func):
"""Will cast results of func as a result, and try to extract
a status_code for the Response object"""
+
@wraps(func)
def wrapper(*args, **kwargs):
status_code = 200
@@ -40,8 +47,12 @@ def jsonify(func):
return result
elif isinstance(result, tuple):
result, status_code = result
- return Response(json.dumps(result, default=scoped_default_handler()),
- mimetype='application/json', status=status_code)
+ return Response(
+ json.dumps(result, default=scoped_default_handler()),
+ mimetype="application/json",
+ status=status_code,
+ )
+
return wrapper
@@ -49,5 +60,4 @@ def login_user_bundle(user):
login_user(user)
identity_changed.send(current_app, identity=Identity(user.id))
session_identity_loader()
- UserController(user.id).update(
- {'id': user.id}, {'last_seen': datetime.utcnow()})
+ UserController(user.id).update({"id": user.id}, {"last_seen": datetime.utcnow()})
diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py
index b98a005a..592e3cbf 100644
--- a/newspipe/web/views/feed.py
+++ b/newspipe/web/views/feed.py
@@ -4,8 +4,15 @@ from datetime import datetime, timedelta
from sqlalchemy import desc
from werkzeug.exceptions import BadRequest
-from flask import Blueprint, render_template, flash, \
- redirect, request, url_for, make_response
+from flask import (
+ Blueprint,
+ render_template,
+ flash,
+ redirect,
+ request,
+ url_for,
+ make_response,
+)
from flask_babel import gettext
from flask_login import login_required, current_user
from flask_paginate import Pagination, get_page_args
@@ -15,24 +22,30 @@ from lib import misc_utils, utils
from lib.feed_utils import construct_feed_from
from web.lib.view_utils import etag_match
from web.forms import AddFeedForm
-from web.controllers import (UserController, CategoryController,
- FeedController, ArticleController)
+from web.controllers import (
+ UserController,
+ CategoryController,
+ FeedController,
+ ArticleController,
+)
logger = logging.getLogger(__name__)
-feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
-feed_bp = Blueprint('feed', __name__, url_prefix='/feed')
+feeds_bp = Blueprint("feeds", __name__, url_prefix="/feeds")
+feed_bp = Blueprint("feed", __name__, url_prefix="/feed")
-@feeds_bp.route('/', methods=['GET'])
+@feeds_bp.route("/", methods=["GET"])
@login_required
@etag_match
def feeds():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
- return render_template('feeds.html',
- feeds=FeedController(current_user.id).read().order_by('title'),
- unread_article_count=art_contr.count_by_feed(readed=False),
- article_count=art_contr.count_by_feed())
+ return render_template(
+ "feeds.html",
+ feeds=FeedController(current_user.id).read().order_by("title"),
+ unread_article_count=art_contr.count_by_feed(readed=False),
+ article_count=art_contr.count_by_feed(),
+ )
def feed_view(feed_id=None, user_id=None):
@@ -42,15 +55,19 @@ def feed_view(feed_id=None, user_id=None):
if feed.category_id:
category = CategoryController(user_id).get(id=feed.category_id)
filters = {}
- filters['feed_id'] = feed_id
+ filters["feed_id"] = feed_id
articles = ArticleController(user_id).read_light(**filters)
# Server-side pagination
- page, per_page, offset = get_page_args(per_page_parameter='per_page')
- pagination = Pagination(page=page, total=articles.count(),
- css_framework='bootstrap3',
- search=False, record_name='articles',
- per_page=per_page)
+ page, per_page, offset = get_page_args(per_page_parameter="per_page")
+ pagination = Pagination(
+ page=page,
+ total=articles.count(),
+ css_framework="bootstrap3",
+ search=False,
+ record_name="articles",
+ per_page=per_page,
+ )
today = datetime.now()
try:
@@ -65,17 +82,22 @@ def feed_view(feed_id=None, user_id=None):
average = 0
elapsed = today - last_article
- return render_template('feed.html',
- head_titles=[utils.clear_string(feed.title)],
- feed=feed, category=category,
- articles=articles.offset(offset).limit(per_page),
- pagination=pagination,
- first_post_date=first_article,
- end_post_date=last_article,
- average=average, delta=delta, elapsed=elapsed)
-
-
-@feed_bp.route('/<int:feed_id>', methods=['GET'])
+ return render_template(
+ "feed.html",
+ head_titles=[utils.clear_string(feed.title)],
+ feed=feed,
+ category=category,
+ articles=articles.offset(offset).limit(per_page),
+ pagination=pagination,
+ first_post_date=first_article,
+ end_post_date=last_article,
+ average=average,
+ delta=delta,
+ elapsed=elapsed,
+ )
+
+
+@feed_bp.route("/<int:feed_id>", methods=["GET"])
@login_required
@etag_match
def feed(feed_id=None):
@@ -83,7 +105,7 @@ def feed(feed_id=None):
return feed_view(feed_id, current_user.id)
-@feed_bp.route('/public/<int:feed_id>', methods=['GET'])
+@feed_bp.route("/public/<int:feed_id>", methods=["GET"])
@etag_match
def feed_pub(feed_id=None):
"""
@@ -92,90 +114,97 @@ def feed_pub(feed_id=None):
"""
feed = FeedController(None).get(id=feed_id)
if feed.private or not feed.user.is_public_profile:
- return render_template('errors/404.html'), 404
+ return render_template("errors/404.html"), 404
return feed_view(feed_id, None)
-@feed_bp.route('/delete/<feed_id>', methods=['GET'])
+@feed_bp.route("/delete/<feed_id>", methods=["GET"])
@login_required
def delete(feed_id=None):
feed_contr = FeedController(current_user.id)
feed = feed_contr.get(id=feed_id)
feed_contr.delete(feed_id)
- flash(gettext("Feed %(feed_title)s successfully deleted.",
- feed_title=feed.title), 'success')
- return redirect(url_for('home'))
+ flash(
+ gettext("Feed %(feed_title)s successfully deleted.", feed_title=feed.title),
+ "success",
+ )
+ return redirect(url_for("home"))
-@feed_bp.route('/reset_errors/<int:feed_id>', methods=['GET', 'POST'])
+@feed_bp.route("/reset_errors/<int:feed_id>", methods=["GET", "POST"])
@login_required
def reset_errors(feed_id):
feed_contr = FeedController(current_user.id)
feed = feed_contr.get(id=feed_id)
- feed_contr.update({'id': feed_id}, {'error_count': 0, 'last_error': ''})
- flash(gettext('Feed %(feed_title)r successfully updated.',
- feed_title=feed.title), 'success')
- return redirect(request.referrer or url_for('home'))
+ feed_contr.update({"id": feed_id}, {"error_count": 0, "last_error": ""})
+ flash(
+ gettext("Feed %(feed_title)r successfully updated.", feed_title=feed.title),
+ "success",
+ )
+ return redirect(request.referrer or url_for("home"))
-@feed_bp.route('/bookmarklet', methods=['GET', 'POST'])
+@feed_bp.route("/bookmarklet", methods=["GET", "POST"])
@login_required
def bookmarklet():
feed_contr = FeedController(current_user.id)
- url = (request.args if request.method == 'GET' else request.form)\
- .get('url', None)
+ url = (request.args if request.method == "GET" else request.form).get("url", None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
- feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
+ feed_exists = list(feed_contr.read(__or__={"link": url, "site_link": url}))
if feed_exists:
- flash(gettext("Couldn't add feed: feed already exists."),
- "warning")
- return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
+ flash(gettext("Couldn't add feed: feed already exists."), "warning")
+ return redirect(url_for("feed.form", feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
- flash(gettext("Impossible to connect to the address: {}.".format(url)),
- "danger")
- return redirect(url_for('home'))
+ flash(
+ gettext("Impossible to connect to the address: {}.".format(url)), "danger"
+ )
+ return redirect(url_for("home"))
except Exception:
- logger.exception('something bad happened when fetching %r', url)
- return redirect(url_for('home'))
- if not feed.get('link'):
- feed['enabled'] = False
- flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
- " RSS link manually and reactivate this feed"),
- 'warning')
+ logger.exception("something bad happened when fetching %r", url)
+ return redirect(url_for("home"))
+ if not feed.get("link"):
+ feed["enabled"] = False
+ flash(
+ gettext(
+ "Couldn't find a feed url, you'll need to find a Atom or"
+ " RSS link manually and reactivate this feed"
+ ),
+ "warning",
+ )
feed = feed_contr.create(**feed)
- flash(gettext('Feed was successfully created.'), 'success')
+ flash(gettext("Feed was successfully created."), "success")
if feed.enabled and conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, feed.id)
- flash(gettext("Downloading articles for the new feed..."), 'info')
- return redirect(url_for('feed.form', feed_id=feed.id))
+ flash(gettext("Downloading articles for the new feed..."), "info")
+ return redirect(url_for("feed.form", feed_id=feed.id))
-@feed_bp.route('/update/<action>/<int:feed_id>', methods=['GET', 'POST'])
-@feeds_bp.route('/update/<action>', methods=['GET', 'POST'])
+@feed_bp.route("/update/<action>/<int:feed_id>", methods=["GET", "POST"])
+@feeds_bp.route("/update/<action>", methods=["GET", "POST"])
@login_required
def update(action, feed_id=None):
- readed = action == 'read'
- filters = {'readed__ne': readed}
+ readed = action == "read"
+ filters = {"readed__ne": readed}
- nb_days = request.args.get('nb_days', 0, type=int)
+ nb_days = request.args.get("nb_days", 0, type=int)
if nb_days != 0:
- filters['date__lt'] = datetime.now() - timedelta(days=nb_days)
+ filters["date__lt"] = datetime.now() - timedelta(days=nb_days)
if feed_id:
- filters['feed_id'] = feed_id
- ArticleController(current_user.id).update(filters, {'readed': readed})
- flash(gettext('Feed successfully updated.'), 'success')
- return redirect(request.referrer or url_for('home'))
+ filters["feed_id"] = feed_id
+ ArticleController(current_user.id).update(filters, {"readed": readed})
+ flash(gettext("Feed successfully updated."), "success")
+ return redirect(request.referrer or url_for("home"))
-@feed_bp.route('/create', methods=['GET'])
-@feed_bp.route('/edit/<int:feed_id>', methods=['GET'])
+@feed_bp.route("/create", methods=["GET"])
+@feed_bp.route("/edit/<int:feed_id>", methods=["GET"])
@login_required
@etag_match
def form(feed_id=None):
@@ -185,22 +214,28 @@ def form(feed_id=None):
if feed_id is None:
form = AddFeedForm()
form.set_category_choices(categories)
- return render_template('edit_feed.html', action=action,
- head_titles=head_titles, form=form)
+ return render_template(
+ "edit_feed.html", action=action, head_titles=head_titles, form=form
+ )
feed = FeedController(current_user.id).get(id=feed_id)
form = AddFeedForm(obj=feed)
form.set_category_choices(categories)
- action = gettext('Edit feed')
+ action = gettext("Edit feed")
head_titles = [action]
if feed.title:
head_titles.append(feed.title)
- return render_template('edit_feed.html', action=action,
- head_titles=head_titles, categories=categories,
- form=form, feed=feed)
-
-
-@feed_bp.route('/create', methods=['POST'])
-@feed_bp.route('/edit/<int:feed_id>', methods=['POST'])
+ return render_template(
+ "edit_feed.html",
+ action=action,
+ head_titles=head_titles,
+ categories=categories,
+ form=form,
+ feed=feed,
+ )
+
+
+@feed_bp.route("/create", methods=["POST"])
+@feed_bp.route("/edit/<int:feed_id>", methods=["POST"])
@login_required
def process_form(feed_id=None):
form = AddFeedForm()
@@ -208,58 +243,68 @@ def process_form(feed_id=None):
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
- return render_template('edit_feed.html', form=form)
+ return render_template("edit_feed.html", form=form)
existing_feeds = list(feed_contr.read(link=form.link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
- return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
+ return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
# Edit an existing feed
- feed_attr = {'title': form.title.data, 'enabled': form.enabled.data,
- 'link': form.link.data, 'site_link': form.site_link.data,
- 'filters': [], 'category_id': form.category_id.data,
- 'private': form.private.data}
- if not feed_attr['category_id'] or feed_attr['category_id'] == '0':
- del feed_attr['category_id']
-
- for filter_attr in ('type', 'pattern', 'action on', 'action'):
- for i, value in enumerate(
- request.form.getlist(filter_attr.replace(' ', '_'))):
- if i >= len(feed_attr['filters']):
- feed_attr['filters'].append({})
- feed_attr['filters'][i][filter_attr] = value
+ feed_attr = {
+ "title": form.title.data,
+ "enabled": form.enabled.data,
+ "link": form.link.data,
+ "site_link": form.site_link.data,
+ "filters": [],
+ "category_id": form.category_id.data,
+ "private": form.private.data,
+ }
+ if not feed_attr["category_id"] or feed_attr["category_id"] == "0":
+ del feed_attr["category_id"]
+
+ for filter_attr in ("type", "pattern", "action on", "action"):
+ for i, value in enumerate(request.form.getlist(filter_attr.replace(" ", "_"))):
+ if i >= len(feed_attr["filters"]):
+ feed_attr["filters"].append({})
+ feed_attr["filters"][i][filter_attr] = value
if feed_id is not None:
- feed_contr.update({'id': feed_id}, feed_attr)
- flash(gettext('Feed %(feed_title)r successfully updated.',
- feed_title=feed_attr['title']), 'success')
- return redirect(url_for('feed.form', feed_id=feed_id))
+ feed_contr.update({"id": feed_id}, feed_attr)
+ flash(
+ gettext(
+ "Feed %(feed_title)r successfully updated.",
+ feed_title=feed_attr["title"],
+ ),
+ "success",
+ )
+ return redirect(url_for("feed.form", feed_id=feed_id))
# Create a new feed
new_feed = feed_contr.create(**feed_attr)
- flash(gettext('Feed %(feed_title)r successfully created.',
- feed_title=new_feed.title), 'success')
+ flash(
+ gettext("Feed %(feed_title)r successfully created.", feed_title=new_feed.title),
+ "success",
+ )
if conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, new_feed.id)
- flash(gettext("Downloading articles for the new feed..."), 'info')
+ flash(gettext("Downloading articles for the new feed..."), "info")
- return redirect(url_for('feed.form', feed_id=new_feed.id))
+ return redirect(url_for("feed.form", feed_id=new_feed.id))
-@feeds_bp.route('/inactives', methods=['GET'])
+@feeds_bp.route("/inactives", methods=["GET"])
@login_required
def inactives():
"""
List of inactive feeds.
"""
- nb_days = int(request.args.get('nb_days', 365))
+ nb_days = int(request.args.get("nb_days", 365))
inactives = FeedController(current_user.id).get_inactives(nb_days)
- return render_template('inactives.html',
- inactives=inactives, nb_days=nb_days)
+ return render_template("inactives.html", inactives=inactives, nb_days=nb_days)
-@feed_bp.route('/duplicates/<int:feed_id>', methods=['GET'])
+@feed_bp.route("/duplicates/<int:feed_id>", methods=["GET"])
@login_required
def duplicates(feed_id):
"""
@@ -267,40 +312,44 @@ def duplicates(feed_id):
"""
feed, duplicates = FeedController(current_user.id).get_duplicates(feed_id)
if len(duplicates) == 0:
- flash(gettext('No duplicates in the feed "{}".').format(feed.title),
- 'info')
- return redirect(url_for('home'))
- return render_template('duplicates.html', duplicates=duplicates, feed=feed)
+ flash(gettext('No duplicates in the feed "{}".').format(feed.title), "info")
+ return redirect(url_for("home"))
+ return render_template("duplicates.html", duplicates=duplicates, feed=feed)
-@feeds_bp.route('/export', methods=['GET'])
+@feeds_bp.route("/export", methods=["GET"])
@login_required
def export():
"""
Export feeds to OPML.
"""
- include_disabled = request.args.get('includedisabled', '') == 'on'
- include_private = request.args.get('includeprivate', '') == 'on'
- include_exceeded_error_count = request.args. \
- get('includeexceedederrorcount', '') == 'on'
+ include_disabled = request.args.get("includedisabled", "") == "on"
+ include_private = request.args.get("includeprivate", "") == "on"
+ include_exceeded_error_count = (
+ request.args.get("includeexceedederrorcount", "") == "on"
+ )
filter = {}
if not include_disabled:
- filter['enabled'] = True
+ filter["enabled"] = True
if not include_private:
- filter['private'] = False
+ filter["private"] = False
if not include_exceeded_error_count:
- filter['error_count__lt'] = conf.DEFAULT_MAX_ERROR
+ filter["error_count__lt"] = conf.DEFAULT_MAX_ERROR
user = UserController(current_user.id).get(id=current_user.id)
feeds = FeedController(current_user.id).read(**filter)
- categories = {cat.id: cat.dump()
- for cat in CategoryController(user.id).read()}
-
- response = make_response(render_template('opml.xml',
- user=user, feeds=feeds,
- categories=categories,
- now=datetime.now()))
- response.headers['Content-Type'] = 'application/xml'
- response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml'
+ categories = {cat.id: cat.dump() for cat in CategoryController(user.id).read()}
+
+ response = make_response(
+ render_template(
+ "opml.xml",
+ user=user,
+ feeds=feeds,
+ categories=categories,
+ now=datetime.now(),
+ )
+ )
+ response.headers["Content-Type"] = "application/xml"
+ response.headers["Content-Disposition"] = "attachment; filename=feeds.opml"
return response
diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py
index dc7a361a..1f51c55c 100644
--- a/newspipe/web/views/home.py
+++ b/newspipe/web/views/home.py
@@ -2,8 +2,7 @@ import pytz
import logging
from datetime import datetime
-from flask import current_app, render_template, \
- request, flash, url_for, redirect
+from flask import current_app, render_template, request, flash, url_for, redirect
from flask_login import login_required, current_user
from flask_babel import gettext, get_locale
from babel.dates import format_datetime, format_timedelta
@@ -14,107 +13,128 @@ from lib import misc_utils
from web.lib.view_utils import etag_match
from web.views.common import jsonify
-from web.controllers import FeedController, \
- ArticleController, CategoryController
+from web.controllers import FeedController, ArticleController, CategoryController
localize = pytz.utc.localize
logger = logging.getLogger(__name__)
-@current_app.route('/')
+@current_app.route("/")
@login_required
@etag_match
def home():
- return render_template('home.html', cdn=conf.CDN_ADDRESS)
+ return render_template("home.html", cdn=conf.CDN_ADDRESS)
-@current_app.route('/menu')
+@current_app.route("/menu")
@login_required
@etag_match
@jsonify
def get_menu():
now, locale = datetime.now(), get_locale()
categories_order = [0]
- categories = {0: {'name': 'No category', 'id': 0}}
- for cat in CategoryController(current_user.id).read().order_by('name'):
+ categories = {0: {"name": "No category", "id": 0}}
+ for cat in CategoryController(current_user.id).read().order_by("name"):
categories_order.append(cat.id)
categories[cat.id] = cat
unread = ArticleController(current_user.id).count_by_feed(readed=False)
for cat_id in categories:
- categories[cat_id]['unread'] = 0
- categories[cat_id]['feeds'] = []
+ categories[cat_id]["unread"] = 0
+ categories[cat_id]["feeds"] = []
feeds = {feed.id: feed for feed in FeedController(current_user.id).read()}
for feed_id, feed in feeds.items():
- feed['created_rel'] = format_timedelta(feed.created_date - now,
- add_direction=True, locale=locale)
- feed['last_rel'] = format_timedelta(feed.last_retrieved - now,
- add_direction=True, locale=locale)
- feed['created_date'] = format_datetime(localize(feed.created_date),
- locale=locale)
- feed['last_retrieved'] = format_datetime(localize(feed.last_retrieved),
- locale=locale)
- feed['category_id'] = feed.category_id or 0
- feed['unread'] = unread.get(feed.id, 0)
+ feed["created_rel"] = format_timedelta(
+ feed.created_date - now, add_direction=True, locale=locale
+ )
+ feed["last_rel"] = format_timedelta(
+ feed.last_retrieved - now, add_direction=True, locale=locale
+ )
+ feed["created_date"] = format_datetime(
+ localize(feed.created_date), locale=locale
+ )
+ feed["last_retrieved"] = format_datetime(
+ localize(feed.last_retrieved), locale=locale
+ )
+ feed["category_id"] = feed.category_id or 0
+ feed["unread"] = unread.get(feed.id, 0)
if not feed.filters:
- feed['filters'] = []
+ feed["filters"] = []
if feed.icon_url:
- feed['icon_url'] = url_for('icon.icon', url=feed.icon_url)
- categories[feed['category_id']]['unread'] += feed['unread']
- categories[feed['category_id']]['feeds'].append(feed_id)
- return {'feeds': feeds, 'categories': categories,
- 'categories_order': categories_order,
- 'crawling_method': conf.CRAWLING_METHOD,
- 'max_error': conf.DEFAULT_MAX_ERROR,
- 'error_threshold': conf.ERROR_THRESHOLD,
- 'is_admin': current_user.is_admin,
- 'all_unread_count': sum(unread.values())}
+ feed["icon_url"] = url_for("icon.icon", url=feed.icon_url)
+ categories[feed["category_id"]]["unread"] += feed["unread"]
+ categories[feed["category_id"]]["feeds"].append(feed_id)
+ return {
+ "feeds": feeds,
+ "categories": categories,
+ "categories_order": categories_order,
+ "crawling_method": conf.CRAWLING_METHOD,
+ "max_error": conf.DEFAULT_MAX_ERROR,
+ "error_threshold": conf.ERROR_THRESHOLD,
+ "is_admin": current_user.is_admin,
+ "all_unread_count": sum(unread.values()),
+ }
def _get_filters(in_dict):
filters = {}
- query = in_dict.get('query')
+ query = in_dict.get("query")
if query:
- search_title = in_dict.get('search_title') == 'true'
- search_content = in_dict.get('search_content') == 'true'
+ search_title = in_dict.get("search_title") == "true"
+ search_content = in_dict.get("search_content") == "true"
if search_title:
- filters['title__ilike'] = "%%%s%%" % query
+ filters["title__ilike"] = "%%%s%%" % query
if search_content:
- filters['content__ilike'] = "%%%s%%" % query
+ filters["content__ilike"] = "%%%s%%" % query
if len(filters) == 0:
- filters['title__ilike'] = "%%%s%%" % query
+ filters["title__ilike"] = "%%%s%%" % query
if len(filters) > 1:
filters = {"__or__": filters}
- if in_dict.get('filter') == 'unread':
- filters['readed'] = False
- elif in_dict.get('filter') == 'liked':
- filters['like'] = True
- filter_type = in_dict.get('filter_type')
- if filter_type in {'feed_id', 'category_id'} and in_dict.get('filter_id'):
- filters[filter_type] = int(in_dict['filter_id']) or None
+ if in_dict.get("filter") == "unread":
+ filters["readed"] = False
+ elif in_dict.get("filter") == "liked":
+ filters["like"] = True
+ filter_type = in_dict.get("filter_type")
+ if filter_type in {"feed_id", "category_id"} and in_dict.get("filter_id"):
+ filters[filter_type] = int(in_dict["filter_id"]) or None
return filters
@jsonify
def _articles_to_json(articles, fd_hash=None):
now, locale = datetime.now(), get_locale()
- fd_hash = {feed.id: {'title': feed.title,
- 'icon_url': url_for('icon.icon', url=feed.icon_url)
- if feed.icon_url else None}
- for feed in FeedController(current_user.id).read()}
-
- return {'articles': [{'title': art.title, 'liked': art.like,
- 'read': art.readed, 'article_id': art.id, 'selected': False,
- 'feed_id': art.feed_id, 'category_id': art.category_id or 0,
- 'feed_title': fd_hash[art.feed_id]['title'] if fd_hash else None,
- 'icon_url': fd_hash[art.feed_id]['icon_url'] if fd_hash else None,
- 'date': format_datetime(localize(art.date), locale=locale),
- 'rel_date': format_timedelta(art.date - now,
- threshold=1.1, add_direction=True,
- locale=locale)}
- for art in articles.limit(1000)]}
-
-
-@current_app.route('/middle_panel')
+ fd_hash = {
+ feed.id: {
+ "title": feed.title,
+ "icon_url": url_for("icon.icon", url=feed.icon_url)
+ if feed.icon_url
+ else None,
+ }
+ for feed in FeedController(current_user.id).read()
+ }
+
+ return {
+ "articles": [
+ {
+ "title": art.title,
+ "liked": art.like,
+ "read": art.readed,
+ "article_id": art.id,
+ "selected": False,
+ "feed_id": art.feed_id,
+ "category_id": art.category_id or 0,
+ "feed_title": fd_hash[art.feed_id]["title"] if fd_hash else None,
+ "icon_url": fd_hash[art.feed_id]["icon_url"] if fd_hash else None,
+ "date": format_datetime(localize(art.date), locale=locale),
+ "rel_date": format_timedelta(
+ art.date - now, threshold=1.1, add_direction=True, locale=locale
+ ),
+ }
+ for art in articles.limit(1000)
+ ]
+ }
+
+
+@current_app.route("/middle_panel")
@login_required
@etag_match
def get_middle_panel():
@@ -124,8 +144,8 @@ def get_middle_panel():
return _articles_to_json(articles)
-@current_app.route('/getart/<int:article_id>')
-@current_app.route('/getart/<int:article_id>/<parse>')
+@current_app.route("/getart/<int:article_id>")
+@current_app.route("/getart/<int:article_id>/<parse>")
@login_required
@etag_match
@jsonify
@@ -134,28 +154,29 @@ def get_article(article_id, parse=False):
contr = ArticleController(current_user.id)
article = contr.get(id=article_id)
if not article.readed:
- article['readed'] = True
- contr.update({'id': article_id}, {'readed': True})
- article['category_id'] = article.category_id or 0
+ article["readed"] = True
+ contr.update({"id": article_id}, {"readed": True})
+ article["category_id"] = article.category_id or 0
feed = FeedController(current_user.id).get(id=article.feed_id)
- article['icon_url'] = url_for('icon.icon', url=feed.icon_url) \
- if feed.icon_url else None
- article['date'] = format_datetime(localize(article.date), locale=locale)
+ article["icon_url"] = (
+ url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None
+ )
+ article["date"] = format_datetime(localize(article.date), locale=locale)
return article
-@current_app.route('/mark_all_as_read', methods=['PUT'])
+@current_app.route("/mark_all_as_read", methods=["PUT"])
@login_required
def mark_all_as_read():
filters = _get_filters(request.json)
acontr = ArticleController(current_user.id)
processed_articles = _articles_to_json(acontr.read_light(**filters))
- acontr.update(filters, {'readed': True})
+ acontr.update(filters, {"readed": True})
return processed_articles
-@current_app.route('/fetch', methods=['GET'])
-@current_app.route('/fetch/<int:feed_id>', methods=['GET'])
+@current_app.route("/fetch", methods=["GET"])
+@current_app.route("/fetch/<int:feed_id>", methods=["GET"])
@login_required
def fetch(feed_id=None):
"""
@@ -166,6 +187,11 @@ def fetch(feed_id=None):
misc_utils.fetch(current_user.id, feed_id)
flash(gettext("Downloading articles..."), "info")
else:
- flash(gettext("The manual retrieving of news is only available " +
- "for administrator, on the Heroku platform."), "info")
+ flash(
+ gettext(
+ "The manual retrieving of news is only available "
+ + "for administrator, on the Heroku platform."
+ ),
+ "info",
+ )
return redirect(redirect_url())
diff --git a/newspipe/web/views/icon.py b/newspipe/web/views/icon.py
index 64e54cab..e1de6402 100644
--- a/newspipe/web/views/icon.py
+++ b/newspipe/web/views/icon.py
@@ -3,13 +3,12 @@ from flask import Blueprint, Response, request
from web.controllers import IconController
from web.lib.view_utils import etag_match
-icon_bp = Blueprint('icon', __name__, url_prefix='/icon')
+icon_bp = Blueprint("icon", __name__, url_prefix="/icon")
-@icon_bp.route('/', methods=['GET'])
+@icon_bp.route("/", methods=["GET"])
@etag_match
def icon():
- icon = IconController().get(url=request.args['url'])
- headers = {'Cache-Control': 'max-age=86400',
- 'Content-Type': icon.mimetype}
+ icon = IconController().get(url=request.args["url"])
+ headers = {"Cache-Control": "max-age=86400", "Content-Type": icon.mimetype}
return Response(base64.b64decode(icon.content), headers=headers)
diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py
index 0db76115..809825d3 100644
--- a/newspipe/web/views/session_mgmt.py
+++ b/newspipe/web/views/session_mgmt.py
@@ -4,14 +4,25 @@ import logging
from datetime import datetime
from werkzeug.security import generate_password_hash
from werkzeug.exceptions import NotFound
-from flask import (render_template, flash, session, request,
- url_for, redirect, current_app)
+from flask import (
+ render_template,
+ flash,
+ session,
+ request,
+ url_for,
+ redirect,
+ current_app,
+)
from flask_babel import gettext, lazy_gettext
-from flask_login import LoginManager, logout_user, \
- login_required, current_user
-from flask_principal import (Principal, AnonymousIdentity, UserNeed,
- identity_changed, identity_loaded,
- session_identity_loader)
+from flask_login import LoginManager, logout_user, login_required, current_user
+from flask_principal import (
+ Principal,
+ AnonymousIdentity,
+ UserNeed,
+ identity_changed,
+ identity_loaded,
+ session_identity_loader,
+)
import conf
from web.views.common import admin_role, api_role, login_user_bundle
@@ -24,9 +35,9 @@ Principal(current_app)
login_manager = LoginManager()
login_manager.init_app(current_app)
-login_manager.login_view = 'login'
-login_manager.login_message = lazy_gettext('Please log in to access this page.')
-login_manager.login_message_category = 'info'
+login_manager.login_view = "login"
+login_manager.login_message = lazy_gettext("Please log in to access this page.")
+login_manager.login_message_category = "info"
logger = logging.getLogger(__name__)
@@ -47,67 +58,77 @@ def on_identity_loaded(sender, identity):
@login_manager.user_loader
def load_user(user_id):
- return UserController(user_id, ignore_context=True).get(
- id=user_id, is_active=True)
+ return UserController(user_id, ignore_context=True).get(id=user_id, is_active=True)
+
@current_app.before_request
def before_request():
if current_user.is_authenticated:
UserController(current_user.id).update(
- {'id': current_user.id}, {'last_seen': datetime.utcnow()})
+ {"id": current_user.id}, {"last_seen": datetime.utcnow()}
+ )
+
-@current_app.route('/login', methods=['GET', 'POST'])
+@current_app.route("/login", methods=["GET", "POST"])
def login():
if current_user.is_authenticated:
- return redirect(url_for('home'))
+ return redirect(url_for("home"))
form = SigninForm()
if form.validate_on_submit():
login_user_bundle(form.user)
- return form.redirect('home')
- return render_template('login.html', form=form)
+ return form.redirect("home")
+ return render_template("login.html", form=form)
-@current_app.route('/logout')
+@current_app.route("/logout")
@login_required
def logout():
# Remove the user information from the session
logout_user()
# Remove session keys set by Flask-Principal
- for key in ('identity.name', 'identity.auth_type'):
+ for key in ("identity.name", "identity.auth_type"):
session.pop(key, None)
# Tell Flask-Principal the user is anonymous
identity_changed.send(current_app, identity=AnonymousIdentity())
session_identity_loader()
- return redirect(url_for('login'))
+ return redirect(url_for("login"))
-@current_app.route('/signup', methods=['GET', 'POST'])
+@current_app.route("/signup", methods=["GET", "POST"])
def signup():
if not conf.SELF_REGISTRATION:
- flash(gettext('Self-registration is disabled.'), 'warning')
- return redirect(url_for('home'))
+ flash(gettext("Self-registration is disabled."), "warning")
+ return redirect(url_for("home"))
if current_user.is_authenticated:
- return redirect(url_for('home'))
+ return redirect(url_for("home"))
form = SignupForm()
if form.validate_on_submit():
- user = UserController().create(nickname=form.nickname.data,
- pwdhash=generate_password_hash(form.password.data))
+ user = UserController().create(
+ nickname=form.nickname.data,
+ pwdhash=generate_password_hash(form.password.data),
+ )
# Send the confirmation email
try:
notifications.new_account_notification(user, form.email.data)
except Exception as error:
- flash(gettext('Problem while sending activation email: %(error)s',
- error=error), 'danger')
- return redirect(url_for('home'))
-
- flash(gettext('Your account has been created. '
- 'Check your mail to confirm it.'), 'success')
-
- return redirect(url_for('home'))
-
- return render_template('signup.html', form=form)
+ flash(
+ gettext(
+ "Problem while sending activation email: %(error)s", error=error
+ ),
+ "danger",
+ )
+ return redirect(url_for("home"))
+
+ flash(
+ gettext("Your account has been created. " "Check your mail to confirm it."),
+ "success",
+ )
+
+ return redirect(url_for("home"))
+
+ return render_template("signup.html", form=form)
diff --git a/newspipe/web/views/user.py b/newspipe/web/views/user.py
index 24b73a60..10974947 100644
--- a/newspipe/web/views/user.py
+++ b/newspipe/web/views/user.py
@@ -1,8 +1,7 @@
import string
import random
from datetime import datetime, timedelta
-from flask import (Blueprint, g, render_template, redirect,
- flash, url_for, request)
+from flask import Blueprint, g, render_template, redirect, flash, url_for, request
from flask_babel import gettext
from flask_login import login_required, current_user
from flask_paginate import Pagination, get_page_args
@@ -12,39 +11,47 @@ from notifications import notifications
from lib import misc_utils
from lib.data import import_opml, import_json
from web.lib.user_utils import confirm_token
-from web.controllers import (UserController, FeedController, ArticleController,
- CategoryController, BookmarkController)
+from web.controllers import (
+ UserController,
+ FeedController,
+ ArticleController,
+ CategoryController,
+ BookmarkController,
+)
from web.forms import ProfileForm
-users_bp = Blueprint('users', __name__, url_prefix='/users')
-user_bp = Blueprint('user', __name__, url_prefix='/user')
+users_bp = Blueprint("users", __name__, url_prefix="/users")
+user_bp = Blueprint("user", __name__, url_prefix="/user")
-@user_bp.route('/<string:nickname>', methods=['GET'])
+@user_bp.route("/<string:nickname>", methods=["GET"])
def profile_public(nickname=None):
"""
Display the public profile of the user.
"""
- category_id = int(request.args.get('category_id', 0))
+ category_id = int(request.args.get("category_id", 0))
user_contr = UserController()
user = user_contr.get(nickname=nickname)
if not user.is_public_profile:
if current_user.is_authenticated and current_user.id == user.id:
- flash(gettext('You must set your profile to public.'), 'info')
- return redirect(url_for('user.profile'))
+ flash(gettext("You must set your profile to public."), "info")
+ return redirect(url_for("user.profile"))
filters = {}
- filters['private'] = False
+ filters["private"] = False
if category_id:
- filters['category_id'] = category_id
+ filters["category_id"] = category_id
feeds = FeedController(user.id).read(**filters)
- return render_template('profile_public.html', user=user, feeds=feeds,
- selected_category_id=category_id)
+ return render_template(
+ "profile_public.html", user=user, feeds=feeds, selected_category_id=category_id
+ )
-@user_bp.route('/<string:nickname>/stream', defaults={'per_page': '25'}, methods=['GET'])
+@user_bp.route(
+ "/<string:nickname>/stream", defaults={"per_page": "25"}, methods=["GET"]
+)
def user_stream(per_page, nickname=None):
"""
Display the stream of a user (list of articles of public feed).
@@ -53,76 +60,80 @@ def user_stream(per_page, nickname=None):
user = user_contr.get(nickname=nickname)
if not user.is_public_profile:
if current_user.is_authenticated and current_user.id == user.id:
- flash(gettext('You must set your profile to public.'), 'info')
- return redirect(url_for('user.profile'))
+ flash(gettext("You must set your profile to public."), "info")
+ return redirect(url_for("user.profile"))
- category_id = int(request.args.get('category_id', 0))
+ category_id = int(request.args.get("category_id", 0))
category = CategoryController().read(id=category_id).first()
# Load the public feeds
filters = {}
- filters['private'] = False
+ filters["private"] = False
if category_id:
- filters['category_id'] = category_id
+ filters["category_id"] = category_id
feeds = FeedController().read(**filters).all()
# Re-initializes the filters to load the articles
filters = {}
- filters['feed_id__in'] = [feed.id for feed in feeds]
+ filters["feed_id__in"] = [feed.id for feed in feeds]
if category:
- filters['category_id'] = category_id
+ filters["category_id"] = category_id
articles = ArticleController(user.id).read_light(**filters)
# Server-side pagination
- page, per_page, offset = get_page_args(per_page_parameter='per_page')
- pagination = Pagination(page=page, total=articles.count(),
- css_framework='bootstrap3',
- search=False, record_name='articles',
- per_page=per_page)
-
- return render_template('user_stream.html', user=user,
- articles=articles.offset(offset).limit(per_page),
- category=category,
- pagination=pagination)
-
-
-@user_bp.route('/management', methods=['GET', 'POST'])
+ page, per_page, offset = get_page_args(per_page_parameter="per_page")
+ pagination = Pagination(
+ page=page,
+ total=articles.count(),
+ css_framework="bootstrap3",
+ search=False,
+ record_name="articles",
+ per_page=per_page,
+ )
+
+ return render_template(
+ "user_stream.html",
+ user=user,
+ articles=articles.offset(offset).limit(per_page),
+ category=category,
+ pagination=pagination,
+ )
+
+
+@user_bp.route("/management", methods=["GET", "POST"])
@login_required
def management():
"""
Display the management page.
"""
- if request.method == 'POST':
- if None != request.files.get('opmlfile', None):
+ if request.method == "POST":
+ if None != request.files.get("opmlfile", None):
# Import an OPML file
- data = request.files.get('opmlfile', None)
+ data = request.files.get("opmlfile", None)
if not misc_utils.allowed_file(data.filename):
- flash(gettext('File not allowed.'), 'danger')
+ flash(gettext("File not allowed."), "danger")
else:
try:
nb = import_opml(current_user.nickname, data.read())
if conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, None)
- flash(str(nb) + ' ' + gettext('feeds imported.'),
- "success")
- flash(gettext("Downloading articles..."), 'info')
+ flash(str(nb) + " " + gettext("feeds imported."), "success")
+ flash(gettext("Downloading articles..."), "info")
except:
- flash(gettext("Impossible to import the new feeds."),
- "danger")
- elif None != request.files.get('jsonfile', None):
+ flash(gettext("Impossible to import the new feeds."), "danger")
+ elif None != request.files.get("jsonfile", None):
# Import an account
- data = request.files.get('jsonfile', None)
+ data = request.files.get("jsonfile", None)
if not misc_utils.allowed_file(data.filename):
- flash(gettext('File not allowed.'), 'danger')
+ flash(gettext("File not allowed."), "danger")
else:
try:
nb = import_json(current_user.nickname, data.read())
- flash(gettext('Account imported.'), "success")
+ flash(gettext("Account imported."), "success")
except:
- flash(gettext("Impossible to import the account."),
- "danger")
+ flash(gettext("Impossible to import the account."), "danger")
else:
- flash(gettext('File not allowed.'), 'danger')
+ flash(gettext("File not allowed."), "danger")
nb_feeds = FeedController(current_user.id).read().count()
art_contr = ArticleController(current_user.id)
@@ -130,14 +141,18 @@ def management():
nb_unread_articles = art_contr.read(readed=False).count()
nb_categories = CategoryController(current_user.id).read().count()
nb_bookmarks = BookmarkController(current_user.id).read().count()
- return render_template('management.html', user=current_user,
- nb_feeds=nb_feeds, nb_articles=nb_articles,
- nb_unread_articles=nb_unread_articles,
- nb_categories=nb_categories,
- nb_bookmarks=nb_bookmarks)
-
-
-@user_bp.route('/profile', methods=['GET', 'POST'])
+ return render_template(
+ "management.html",
+ user=current_user,
+ nb_feeds=nb_feeds,
+ nb_articles=nb_articles,
+ nb_unread_articles=nb_unread_articles,
+ nb_categories=nb_categories,
+ nb_bookmarks=nb_bookmarks,
+ )
+
+
+@user_bp.route("/profile", methods=["GET", "POST"])
@login_required
def profile():
"""
@@ -147,44 +162,54 @@ def profile():
user = user_contr.get(id=current_user.id)
form = ProfileForm()
- if request.method == 'POST':
+ if request.method == "POST":
if form.validate():
try:
- user_contr.update({'id': current_user.id},
- {'nickname': form.nickname.data,
- 'password': form.password.data,
- 'automatic_crawling': form.automatic_crawling.data,
- 'is_public_profile': form.is_public_profile.data,
- 'bio': form.bio.data,
- 'webpage': form.webpage.data,
- 'twitter': form.twitter.data})
+ user_contr.update(
+ {"id": current_user.id},
+ {
+ "nickname": form.nickname.data,
+ "password": form.password.data,
+ "automatic_crawling": form.automatic_crawling.data,
+ "is_public_profile": form.is_public_profile.data,
+ "bio": form.bio.data,
+ "webpage": form.webpage.data,
+ "twitter": form.twitter.data,
+ },
+ )
except Exception as error:
- flash(gettext('Problem while updating your profile: '
- '%(error)s', error=error), 'danger')
+ flash(
+ gettext(
+ "Problem while updating your profile: " "%(error)s", error=error
+ ),
+ "danger",
+ )
else:
- flash(gettext('User %(nick)s successfully updated',
- nick=user.nickname), 'success')
- return redirect(url_for('user.profile'))
+ flash(
+ gettext("User %(nick)s successfully updated", nick=user.nickname),
+ "success",
+ )
+ return redirect(url_for("user.profile"))
else:
- return render_template('profile.html', user=user, form=form)
+ return render_template("profile.html", user=user, form=form)
- if request.method == 'GET':
+ if request.method == "GET":
form = ProfileForm(obj=user)
- return render_template('profile.html', user=user, form=form)
+ return render_template("profile.html", user=user, form=form)
-@user_bp.route('/delete_account', methods=['GET'])
+@user_bp.route("/delete_account", methods=["GET"])
@login_required
def delete_account():
"""
Delete the account of the user (with all its data).
"""
UserController(current_user.id).delete(current_user.id)
- flash(gettext('Your account has been deleted.'), 'success')
- return redirect(url_for('login'))
+ flash(gettext("Your account has been deleted."), "success")
+ return redirect(url_for("login"))
-@user_bp.route('/confirm_account/<string:token>', methods=['GET'])
+@user_bp.route("/confirm_account/<string:token>", methods=["GET"])
def confirm_account(token=None):
"""
Confirm the account of a user.
@@ -196,8 +221,8 @@ def confirm_account(token=None):
if nickname:
user = user_contr.read(nickname=nickname).first()
if user is not None:
- user_contr.update({'id': user.id}, {'is_active': True})
- flash(gettext('Your account has been confirmed.'), 'success')
+ user_contr.update({"id": user.id}, {"is_active": True})
+ flash(gettext("Your account has been confirmed."), "success")
else:
- flash(gettext('Impossible to confirm this account.'), 'danger')
- return redirect(url_for('login'))
+ flash(gettext("Impossible to confirm this account."), "danger")
+ return redirect(url_for("login"))
diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py
index d587bd09..1fde12c7 100644
--- a/newspipe/web/views/views.py
+++ b/newspipe/web/views/views.py
@@ -2,8 +2,7 @@ import sys
import logging
import operator
from datetime import datetime, timedelta
-from flask import (request, render_template, flash,
- url_for, redirect, current_app)
+from flask import request, render_template, flash, url_for, redirect, current_app
from flask_babel import gettext
from sqlalchemy import desc
@@ -20,26 +19,26 @@ logger = logging.getLogger(__name__)
def authentication_required(error):
if API_ROOT in request.url:
return error
- flash(gettext('Authentication required.'), 'info')
- return redirect(url_for('login'))
+ flash(gettext("Authentication required."), "info")
+ return redirect(url_for("login"))
@current_app.errorhandler(403)
def authentication_failed(error):
if API_ROOT in request.url:
return error
- flash(gettext('Forbidden.'), 'danger')
- return redirect(url_for('login'))
+ flash(gettext("Forbidden."), "danger")
+ return redirect(url_for("login"))
@current_app.errorhandler(404)
def page_not_found(error):
- return render_template('errors/404.html'), 404
+ return render_template("errors/404.html"), 404
@current_app.errorhandler(500)
def internal_server_error(error):
- return render_template('errors/500.html'), 500
+ return render_template("errors/500.html"), 500
@current_app.errorhandler(AssertionError)
@@ -47,7 +46,7 @@ def handle_sqlalchemy_assertion_error(error):
return error.args[0], 400
-@current_app.route('/popular', methods=['GET'])
+@current_app.route("/popular", methods=["GET"])
@etag_match
def popular():
"""
@@ -57,11 +56,12 @@ def popular():
# 'not_created_before'
# ie: not_added_before = date_last_added_feed - nb_days
try:
- nb_days = int(request.args.get('nb_days', 365))
+ nb_days = int(request.args.get("nb_days", 365))
except ValueError:
nb_days = 10000
- last_added_feed = FeedController().read().\
- order_by(desc('created_date')).limit(1).all()
+ last_added_feed = (
+ FeedController().read().order_by(desc("created_date")).limit(1).all()
+ )
if last_added_feed:
date_last_added_feed = last_added_feed[0].created_date
else:
@@ -69,25 +69,27 @@ def popular():
not_added_before = date_last_added_feed - timedelta(days=nb_days)
filters = {}
- filters['created_date__gt'] = not_added_before
- filters['private'] = False
- filters['error_count__lt'] = conf.DEFAULT_MAX_ERROR
+ filters["created_date__gt"] = not_added_before
+ filters["private"] = False
+ filters["error_count__lt"] = conf.DEFAULT_MAX_ERROR
feeds = FeedController().count_by_link(**filters)
- sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1),
- reverse=True)
- return render_template('popular.html', popular=sorted_feeds)
+ sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True)
+ return render_template("popular.html", popular=sorted_feeds)
-@current_app.route('/about', methods=['GET'])
+@current_app.route("/about", methods=["GET"])
@etag_match
def about():
- return render_template('about.html', contact=ADMIN_EMAIL)
+ return render_template("about.html", contact=ADMIN_EMAIL)
-@current_app.route('/about/more', methods=['GET'])
+
+@current_app.route("/about/more", methods=["GET"])
@etag_match
def about_more():
- return render_template('about_more.html',
- newspipe_version=__version__.split()[1],
- registration=[conf.SELF_REGISTRATION and 'Open' or 'Closed'][0],
- python_version="{}.{}.{}".format(*sys.version_info[:3]),
- nb_users=UserController().read().count())
+ return render_template(
+ "about_more.html",
+ newspipe_version=__version__.split()[1],
+ registration=[conf.SELF_REGISTRATION and "Open" or "Closed"][0],
+ python_version="{}.{}.{}".format(*sys.version_info[:3]),
+ nb_users=UserController().read().count(),
+ )
bgstack15