aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--newspipe/bootstrap.py4
-rwxr-xr-xnewspipe/commands.py19
-rw-r--r--newspipe/controllers/abstract.py2
-rw-r--r--newspipe/controllers/feed.py4
-rw-r--r--newspipe/crawler/default_crawler.py14
-rw-r--r--newspipe/lib/article_utils.py8
-rw-r--r--newspipe/lib/data.py10
-rwxr-xr-xnewspipe/lib/misc_utils.py5
-rw-r--r--newspipe/models/article.py20
-rw-r--r--newspipe/models/feed.py11
-rw-r--r--newspipe/models/tag.py5
-rw-r--r--newspipe/notifications/notifications.py8
-rw-r--r--newspipe/web/views/api/v2/common.py4
-rw-r--r--newspipe/web/views/feed.py4
-rw-r--r--newspipe/web/views/home.py22
-rw-r--r--newspipe/web/views/icon.py7
-rw-r--r--newspipe/web/views/session_mgmt.py4
-rw-r--r--newspipe/web/views/views.py4
18 files changed, 83 insertions, 72 deletions
diff --git a/newspipe/bootstrap.py b/newspipe/bootstrap.py
index dc233f09..9ca81d95 100644
--- a/newspipe/bootstrap.py
+++ b/newspipe/bootstrap.py
@@ -69,7 +69,9 @@ db = SQLAlchemy(application)
migrate = Migrate(application, db)
-talisman = Talisman(application, content_security_policy=application.config["CONTENT_SECURITY_POLICY"])
+talisman = Talisman(
+ application, content_security_policy=application.config["CONTENT_SECURITY_POLICY"]
+)
babel = Babel(application)
diff --git a/newspipe/commands.py b/newspipe/commands.py
index 12ddb486..2eba5d15 100755
--- a/newspipe/commands.py
+++ b/newspipe/commands.py
@@ -34,8 +34,8 @@ def db_create():
@application.cli.command("create_admin")
-@click.option('--nickname', default='admin', help='Nickname')
-@click.option('--password', default='password', help='Password')
+@click.option("--nickname", default="admin", help="Nickname")
+@click.option("--password", default="password", help="Password")
def create_admin(nickname, password):
"Will create an admin user."
admin = {
@@ -53,7 +53,7 @@ def create_admin(nickname, password):
@application.cli.command("delete_user")
-@click.option('--user-id', required=True, help='Id of the user to delete.')
+@click.option("--user-id", required=True, help="Id of the user to delete.")
def delete_user(user_id=None):
"Delete the user with the id specified in the command line."
try:
@@ -64,7 +64,7 @@ def delete_user(user_id=None):
@application.cli.command("delete_inactive_users")
-@click.option('--last-seen', default=6, help='Number of months since last seen.')
+@click.option("--last-seen", default=6, help="Number of months since last seen.")
def delete_inactive_users(last_seen):
"Delete inactive users (inactivity is given in parameter and specified in number of months)."
filter = {}
@@ -81,7 +81,7 @@ def delete_inactive_users(last_seen):
@application.cli.command("disable_inactive_users")
-@click.option('--last-seen', default=6, help='Number of months since last seen.')
+@click.option("--last-seen", default=6, help="Number of months since last seen.")
def disable_inactive_users(last_seen):
"Disable inactive users (inactivity is given in parameter and specified in number of months)."
filter = {}
@@ -101,10 +101,11 @@ def disable_inactive_users(last_seen):
@application.cli.command("delete_read_articles")
def delete_read_articles():
- "Delete read articles retrieved since more than 15 days ago."
+ "Delete read articles (and not liked) retrieved since more than 15 days ago."
filter = {}
filter["user_id__ne"] = 1
- #filter["readed"] = True # temporary comment
+ filter["readed"] = True
+ filter["like"] = False
filter["retrieved_date__lt"] = date.today() - relativedelta(days=15)
articles = ArticleController().read(**filter).limit(5000)
for article in articles:
@@ -130,8 +131,8 @@ def fix_article_entry_id():
@application.cli.command("fetch_asyncio")
-@click.option('--user-id', default=None, help='Id of the user')
-@click.option('--feed-id', default=None, help='If of the feed')
+@click.option("--user-id", default=None, help="Id of the user")
+@click.option("--feed-id", default=None, help="If of the feed")
def fetch_asyncio(user_id=None, feed_id=None):
"Crawl the feeds with asyncio."
import asyncio
diff --git a/newspipe/controllers/abstract.py b/newspipe/controllers/abstract.py
index 6401763b..0d422d43 100644
--- a/newspipe/controllers/abstract.py
+++ b/newspipe/controllers/abstract.py
@@ -63,7 +63,7 @@ class AbstractController:
return db_filters
def _get(self, **filters):
- """ Will add the current user id if that one is not none (in which case
+ """Will add the current user id if that one is not none (in which case
the decision has been made in the code that the query shouldn't be user
dependent) and the user is not an admin and the filters doesn't already
contains a filter for that user.
diff --git a/newspipe/controllers/feed.py b/newspipe/controllers/feed.py
index 72f9b180..513c5bd0 100644
--- a/newspipe/controllers/feed.py
+++ b/newspipe/controllers/feed.py
@@ -88,9 +88,7 @@ class FeedController(AbstractController):
icon_contr.create(**{"url": attrs["icon_url"]})
def create(self, **attrs):
- assert (
- 'link' in attrs
- ), "A feed must have a link."
+ assert "link" in attrs, "A feed must have a link."
self._ensure_icon(attrs)
return super().create(**attrs)
diff --git a/newspipe/crawler/default_crawler.py b/newspipe/crawler/default_crawler.py
index 10ac1cc3..53b960ba 100644
--- a/newspipe/crawler/default_crawler.py
+++ b/newspipe/crawler/default_crawler.py
@@ -112,8 +112,7 @@ async def parse_feed(user, feed):
async def insert_articles(queue, nḅ_producers=1):
- """Consumer coroutines.
- """
+ """Consumer coroutines."""
nb_producers_done = 0
while True:
item = await queue.get()
@@ -172,14 +171,19 @@ async def retrieve_feed(queue, users, feed_id=None):
filters["last_retrieved__lt"] = datetime.now() - timedelta(
minutes=application.config["FEED_REFRESH_INTERVAL"]
)
- #feeds = FeedController().read(**filters).all()
- feeds = [] # temporary fix for: sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL SYSCALL error: EOF detected
+ # feeds = FeedController().read(**filters).all()
+ feeds = (
+ []
+ ) # temporary fix for: sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL SYSCALL error: EOF detected
for feed in user.feeds:
if not feed.enabled:
continue
if feed.error_count > application.config["DEFAULT_MAX_ERROR"]:
continue
- if feed.last_retrieved > (datetime.now() - timedelta(minutes=application.config["FEED_REFRESH_INTERVAL"])):
+ if feed.last_retrieved > (
+ datetime.now()
+ - timedelta(minutes=application.config["FEED_REFRESH_INTERVAL"])
+ ):
continue
if None is feed_id or (feed_id and feed_id == feed.id):
feeds.append(feed)
diff --git a/newspipe/lib/article_utils.py b/newspipe/lib/article_utils.py
index 90b97752..6bf289c1 100644
--- a/newspipe/lib/article_utils.py
+++ b/newspipe/lib/article_utils.py
@@ -18,9 +18,9 @@ PROCESSED_DATE_KEYS = {"published", "created", "updated"}
def extract_id(entry):
- """ extract a value from an entry that will identify it among the other of
+ """extract a value from an entry that will identify it among the other of
that feed"""
- entry_id = 'undefined'
+ entry_id = "undefined"
try:
entry_id = entry.get("entry_id") or entry.get("id") or entry["link"]
except:
@@ -49,7 +49,9 @@ async def construct_article(entry, feed, fields=None, fetch=True):
timezone.utc
)
except ParserError:
- logger.exception("Error when parsing date: {}".format(entry[date_key]))
+ logger.exception(
+ "Error when parsing date: {}".format(entry[date_key])
+ )
except Exception as e:
pass
else:
diff --git a/newspipe/lib/data.py b/newspipe/lib/data.py
index feca38a1..689e6cb0 100644
--- a/newspipe/lib/data.py
+++ b/newspipe/lib/data.py
@@ -112,7 +112,7 @@ def import_json(nickname, json_content):
nb_feeds, nb_articles = 0, 0
# Create feeds:
for feed in json_account:
- if 'link' not in feed.keys() or feed['link'] is None:
+ if "link" not in feed.keys() or feed["link"] is None:
continue
if (
None
@@ -134,7 +134,7 @@ def import_json(nickname, json_content):
db.session.commit()
# Create articles:
for feed in json_account:
- if 'link' not in feed.keys() or feed['link'] is None:
+ if "link" not in feed.keys() or feed["link"] is None:
continue
user_feed = Feed.query.filter(
Feed.user_id == user.id, Feed.link == feed["link"]
@@ -201,8 +201,7 @@ def export_json(user):
def import_pinboard_json(user, json_content):
- """Import bookmarks from a pinboard JSON export.
- """
+ """Import bookmarks from a pinboard JSON export."""
bookmark_contr = BookmarkController(user.id)
BookmarkTagController(user.id)
bookmarks = json.loads(json_content.decode("utf-8"))
@@ -234,8 +233,7 @@ def import_pinboard_json(user, json_content):
def export_bookmarks(user):
- """Export all bookmarks of a user (compatible with Pinboard).
- """
+ """Export all bookmarks of a user (compatible with Pinboard)."""
bookmark_contr = BookmarkController(user.id)
bookmarks = bookmark_contr.read()
export = []
diff --git a/newspipe/lib/misc_utils.py b/newspipe/lib/misc_utils.py
index b2438b82..2bb8f1c2 100755
--- a/newspipe/lib/misc_utils.py
+++ b/newspipe/lib/misc_utils.py
@@ -101,11 +101,12 @@ def fetch(id, feed_id=None):
The default crawler ("asyncio") is launched with the manager.
"""
env = os.environ.copy()
- env['FLASK_APP'] = 'runserver.py'
+ env["FLASK_APP"] = "runserver.py"
cmd = [
sys.exec_prefix + "/bin/flask",
"fetch_asyncio",
- "--user-id", str(id),
+ "--user-id",
+ str(id),
]
if feed_id:
cmd.extend(["--feed-id", str(feed_id)])
diff --git a/newspipe/models/article.py b/newspipe/models/article.py
index a8c247c0..1da56c64 100644
--- a/newspipe/models/article.py
+++ b/newspipe/models/article.py
@@ -64,17 +64,15 @@ class Article(db.Model, RightMixin):
tags = association_proxy("tag_objs", "text")
__table_args__ = (
- ForeignKeyConstraint([user_id], ['user.id'], ondelete='CASCADE'),
- ForeignKeyConstraint([feed_id], ['feed.id'], ondelete='CASCADE'),
- ForeignKeyConstraint([category_id], ['category.id'],
- ondelete='CASCADE'),
- Index('ix_article_eid_cid_uid', user_id, category_id, entry_id),
- Index('ix_article_retrdate', retrieved_date),
-
- Index('user_id'),
- Index('user_id', 'category_id'),
- Index('user_id', 'feed_id'),
- Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id)
+ ForeignKeyConstraint([user_id], ["user.id"], ondelete="CASCADE"),
+ ForeignKeyConstraint([feed_id], ["feed.id"], ondelete="CASCADE"),
+ ForeignKeyConstraint([category_id], ["category.id"], ondelete="CASCADE"),
+ Index("ix_article_eid_cid_uid", user_id, category_id, entry_id),
+ Index("ix_article_retrdate", retrieved_date),
+ Index("user_id"),
+ Index("user_id", "category_id"),
+ Index("user_id", "feed_id"),
+ Index("ix_article_uid_fid_eid", user_id, feed_id, entry_id),
)
# api whitelists
diff --git a/newspipe/models/feed.py b/newspipe/models/feed.py
index 12622285..7e8d92ae 100644
--- a/newspipe/models/feed.py
+++ b/newspipe/models/feed.py
@@ -73,12 +73,11 @@ class Feed(db.Model, RightMixin):
)
__table_args__ = (
- ForeignKeyConstraint([user_id], ['user.id'], ondelete='CASCADE'),
- ForeignKeyConstraint([category_id], ['category.id'],
- ondelete='CASCADE'),
- ForeignKeyConstraint([icon_url], ['icon.url']),
- Index('ix_feed_uid', user_id),
- Index('ix_feed_uid_cid', user_id, category_id),
+ ForeignKeyConstraint([user_id], ["user.id"], ondelete="CASCADE"),
+ ForeignKeyConstraint([category_id], ["category.id"], ondelete="CASCADE"),
+ ForeignKeyConstraint([icon_url], ["icon.url"]),
+ Index("ix_feed_uid", user_id),
+ Index("ix_feed_uid_cid", user_id, category_id),
)
# idx_feed_uid_cid = Index("user_id", "category_id")
diff --git a/newspipe/models/tag.py b/newspipe/models/tag.py
index 0a0fe58c..02a6d29f 100644
--- a/newspipe/models/tag.py
+++ b/newspipe/models/tag.py
@@ -23,9 +23,8 @@ class ArticleTag(db.Model):
self.text = text
__table_args__ = (
- ForeignKeyConstraint([article_id], ['article.id'],
- ondelete='CASCADE'),
- Index('ix_articletag_aid', article_id),
+ ForeignKeyConstraint([article_id], ["article.id"], ondelete="CASCADE"),
+ Index("ix_articletag_aid", article_id),
)
diff --git a/newspipe/notifications/notifications.py b/newspipe/notifications/notifications.py
index 6d5b9fb4..eabcd115 100644
--- a/newspipe/notifications/notifications.py
+++ b/newspipe/notifications/notifications.py
@@ -46,7 +46,9 @@ def new_account_notification(user, email):
)
emails.send(
- to=email, subject="[Newspipe] Account creation", plaintext=plaintext,
+ to=email,
+ subject="[Newspipe] Account creation",
+ plaintext=plaintext,
)
@@ -56,5 +58,7 @@ def new_password_notification(user, password):
"""
plaintext = render_template("emails/new_password.txt", user=user, password=password)
emails.send(
- to=user.email, subject="[Newspipe] New password", plaintext=plaintext,
+ to=user.email,
+ subject="[Newspipe] New password",
+ plaintext=plaintext,
)
diff --git a/newspipe/web/views/api/v2/common.py b/newspipe/web/views/api/v2/common.py
index d785dda5..36817118 100644
--- a/newspipe/web/views/api/v2/common.py
+++ b/newspipe/web/views/api/v2/common.py
@@ -112,9 +112,7 @@ class PyAggAbstractResource(Resource):
if not default:
for value in in_values:
- parser.add_argument(
- value, location="json", default=in_values[value]
- )
+ parser.add_argument(value, location="json", default=in_values[value])
return parser.parse_args(req=request.args, strict=strict)
diff --git a/newspipe/web/views/feed.py b/newspipe/web/views/feed.py
index 73d68c94..10d0b527 100644
--- a/newspipe/web/views/feed.py
+++ b/newspipe/web/views/feed.py
@@ -242,7 +242,9 @@ def process_form(feed_id=None):
if not form.validate():
return render_template("edit_feed.html", form=form)
- existing_feeds = list(feed_contr.read(link=form.link.data, site_link=form.site_link.data))
+ existing_feeds = list(
+ feed_contr.read(link=form.link.data, site_link=form.site_link.data)
+ )
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
diff --git a/newspipe/web/views/home.py b/newspipe/web/views/home.py
index a74c5ec2..e7ee603c 100644
--- a/newspipe/web/views/home.py
+++ b/newspipe/web/views/home.py
@@ -21,8 +21,7 @@ logger = logging.getLogger(__name__)
@current_app.route("/")
@login_required
def home():
- """Displays the home page of the connected user.
- """
+ """Displays the home page of the connected user."""
filters = _get_filters(request.args)
category_contr = CategoryController(current_user.id)
@@ -76,17 +75,14 @@ def home():
search_title=search_title,
search_content=search_content,
):
- return (
- "?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s"
- % (
- filter_,
- limit,
- feed,
- 1 if liked else 0,
- query,
- search_title,
- search_content,
- )
+ return "?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s" % (
+ filter_,
+ limit,
+ feed,
+ 1 if liked else 0,
+ query,
+ search_title,
+ search_content,
)
return render_template(
diff --git a/newspipe/web/views/icon.py b/newspipe/web/views/icon.py
index 62c39c3e..80a87448 100644
--- a/newspipe/web/views/icon.py
+++ b/newspipe/web/views/icon.py
@@ -17,4 +17,9 @@ def icon():
return Response(base64.b64decode(icon.content), headers=headers)
except:
headers = {"Cache-Control": "max-age=86400", "Content-Type": "image/gif"}
- return Response(base64.b64decode("R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="), headers=headers)
+ return Response(
+ base64.b64decode(
+ "R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
+ ),
+ headers=headers,
+ )
diff --git a/newspipe/web/views/session_mgmt.py b/newspipe/web/views/session_mgmt.py
index 5e0ac64d..90f3c141 100644
--- a/newspipe/web/views/session_mgmt.py
+++ b/newspipe/web/views/session_mgmt.py
@@ -58,7 +58,9 @@ def on_identity_loaded(sender, identity):
@login_manager.user_loader
def load_user(user_id):
try:
- return UserController(user_id, ignore_context=True).get(id=user_id, is_active=True)
+ return UserController(user_id, ignore_context=True).get(
+ id=user_id, is_active=True
+ )
except NotFound:
pass
diff --git a/newspipe/web/views/views.py b/newspipe/web/views/views.py
index a62eb706..d896c97b 100644
--- a/newspipe/web/views/views.py
+++ b/newspipe/web/views/views.py
@@ -107,5 +107,7 @@ def about_more():
],
python_version="{}.{}.{}".format(*sys.version_info[:3]),
nb_users=UserController().read().count(),
- content_security_policy=talisman._parse_policy(talisman.content_security_policy),
+ content_security_policy=talisman._parse_policy(
+ talisman.content_security_policy
+ ),
)
bgstack15