Refactoring the backend.

10 files changed, 676 insertions, 0 deletions

diff --git a/newspipe/models/__init__.py b/newspipe/models/__init__.py
new file mode 100644
index 00000000..a58a7ad5
--- /dev/null
+++ b/newspipe/models/__init__.py
@@ -0,0 +1,98 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from .feed import Feed
+from .role import Role
+from .user import User
+from .article import Article
+from .icon import Icon
+from .category import Category
+from .tag import BookmarkTag
+from .tag import ArticleTag
+from .bookmark import Bookmark
+
+__all__ = [
+    "Feed",
+    "Role",
+    "User",
+    "Article",
+    "Icon",
+    "Category",
+    "Bookmark",
+    "ArticleTag",
+    "BookmarkTag",
+]
+
+import os
+
+from sqlalchemy.engine import reflection
+from sqlalchemy.schema import (
+    MetaData,
+    Table,
+    DropTable,
+    ForeignKeyConstraint,
+    DropConstraint,
+)
+
+
+def db_empty(db):
+    "Will drop every datas stocked in db."
+    # From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
+    conn = db.engine.connect()
+
+    # the transaction only applies if the DB supports
+    # transactional DDL, i.e. Postgresql, MS SQL Server
+    trans = conn.begin()
+
+    inspector = reflection.Inspector.from_engine(db.engine)
+
+    # gather all data first before dropping anything.
+    # some DBs lock after things have been dropped in
+    # a transaction.
+    metadata = MetaData()
+
+    tbs = []
+    all_fks = []
+
+    for table_name in inspector.get_table_names():
+        fks = []
+        for fk in inspector.get_foreign_keys(table_name):
+            if not fk["name"]:
+                continue
+            fks.append(ForeignKeyConstraint((), (), name=fk["name"]))
+        t = Table(table_name, metadata, *fks)
+        tbs.append(t)
+        all_fks.extend(fks)
+
+    for fkc in all_fks:
+        conn.execute(DropConstraint(fkc))
+
+    for table in tbs:
+        conn.execute(DropTable(table))
+
+    trans.commit()
diff --git a/newspipe/models/article.py b/newspipe/models/article.py
new file mode 100644
index 00000000..ecc2352b
--- /dev/null
+++ b/newspipe/models/article.py
@@ -0,0 +1,101 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.5 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2016/10/04 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from newspipe.bootstrap import db
+from datetime import datetime
+from sqlalchemy import Index
+from sqlalchemy.ext.associationproxy import association_proxy
+
+from newspipe.models.right_mixin import RightMixin
+
+
+class Article(db.Model, RightMixin):
+    "Represent an article from a feed."
+    id = db.Column(db.Integer(), primary_key=True)
+    entry_id = db.Column(db.String(), nullable=False)
+    link = db.Column(db.String())
+    title = db.Column(db.String())
+    content = db.Column(db.String())
+    readed = db.Column(db.Boolean(), default=False)
+    like = db.Column(db.Boolean(), default=False)
+    date = db.Column(db.DateTime(), default=datetime.utcnow)
+    updated_date = db.Column(db.DateTime(), default=datetime.utcnow)
+    retrieved_date = db.Column(db.DateTime(), default=datetime.utcnow)
+
+    # foreign keys
+    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
+    feed_id = db.Column(db.Integer(), db.ForeignKey("feed.id"))
+    category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))
+
+    # relationships
+    tag_objs = db.relationship(
+        "ArticleTag",
+        back_populates="article",
+        cascade="all,delete-orphan",
+        lazy=False,
+        foreign_keys="[ArticleTag.article_id]",
+    )
+    tags = association_proxy("tag_objs", "text")
+
+    # indexes
+    # __table_args__ = (
+    #    Index('user_id'),
+    #    Index('user_id', 'category_id'),
+    #    Index('user_id', 'feed_id'),
+    #    Index('ix_article_uid_fid_eid', user_id, feed_id, entry_id)
+    # )
+
+    # api whitelists
+    @staticmethod
+    def _fields_base_write():
+        return {"readed", "like", "feed_id", "category_id"}
+
+    @staticmethod
+    def _fields_base_read():
+        return {
+            "id",
+            "entry_id",
+            "link",
+            "title",
+            "content",
+            "date",
+            "retrieved_date",
+            "user_id",
+            "tags",
+        }
+
+    @staticmethod
+    def _fields_api_write():
+        return {"tags"}
+
+    def __repr__(self):
+        return (
+            "<Article(id=%d, entry_id=%s, title=%r, "
+            "date=%r, retrieved_date=%r)>"
+            % (self.id, self.entry_id, self.title, self.date, self.retrieved_date)
+        )
diff --git a/newspipe/models/bookmark.py b/newspipe/models/bookmark.py
new file mode 100644
index 00000000..c3bc2dea
--- /dev/null
+++ b/newspipe/models/bookmark.py
@@ -0,0 +1,72 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.1 $"
+__date__ = "$Date: 2016/12/07 $"
+__revision__ = "$Date: 2016/12/07 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from newspipe.bootstrap import db
+from datetime import datetime
+from sqlalchemy import desc
+from sqlalchemy.orm import validates
+from sqlalchemy.ext.associationproxy import association_proxy
+
+from newspipe.models.tag import BookmarkTag
+from newspipe.models.right_mixin import RightMixin
+
+
+class Bookmark(db.Model, RightMixin):
+    """
+    Represent a bookmark.
+    """
+
+    id = db.Column(db.Integer(), primary_key=True)
+    href = db.Column(db.String(), default="")
+    title = db.Column(db.String(), default="")
+    description = db.Column(db.String(), default="")
+    shared = db.Column(db.Boolean(), default=False)
+    to_read = db.Column(db.Boolean(), default=False)
+    time = db.Column(db.DateTime(), default=datetime.utcnow)
+    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
+
+    # relationships
+    tags = db.relationship(
+        BookmarkTag,
+        backref="of_bookmark",
+        lazy="dynamic",
+        cascade="all,delete-orphan",
+        order_by=desc(BookmarkTag.text),
+    )
+    tags_proxy = association_proxy("tags", "text")
+
+    @validates("description")
+    def validates_title(self, key, value):
+        return str(value).strip()
+
+    @validates("extended")
+    def validates_description(self, key, value):
+        return str(value).strip()
+
+    def __repr__(self):
+        return "<Bookmark %r>" % (self.href)
diff --git a/newspipe/models/category.py b/newspipe/models/category.py
new file mode 100644
index 00000000..3abb6717
--- /dev/null
+++ b/newspipe/models/category.py
@@ -0,0 +1,28 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from newspipe.bootstrap import db
+from sqlalchemy import Index
+from newspipe.models.right_mixin import RightMixin
+
+
+class Category(db.Model, RightMixin):
+    id = db.Column(db.Integer(), primary_key=True)
+    name = db.Column(db.String())
+
+    # relationships
+    user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
+    feeds = db.relationship("Feed", cascade="all,delete-orphan")
+    articles = db.relationship("Article", cascade="all,delete-orphan")
+
+    # index
+    idx_category_uid = Index("user_id")
+
+    # api whitelists
+    @staticmethod
+    def _fields_base_read():
+        return {"id", "user_id"}
+
+    @staticmethod
+    def _fields_base_write():
+        return {"name"}
diff --git a/newspipe/models/feed.py b/newspipe/models/feed.py
new file mode 100644
index 00000000..fc88c17f
--- /dev/null
+++ b/newspipe/models/feed.py
@@ -0,0 +1,105 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from newspipe.bootstrap import db
+from datetime import datetime
+from sqlalchemy import desc, Index
+from sqlalchemy.orm import validates
+from newspipe.models.right_mixin import RightMixin
+from newspipe.models.article import Article
+
+
+class Feed(db.Model, RightMixin):
+    """
+    Represent a feed.
+    """
+
+    id = db.Column(db.Integer(), primary_key=True)
+    title = db.Column(db.String(), default="")
+    description = db.Column(db.String(), default="FR")
+    link = db.Column(db.String(), nullable=False)
+    site_link = db.Column(db.String(), default="")
+    enabled = db.Column(db.Boolean(), default=True)
+    created_date = db.Column(db.DateTime(), default=datetime.utcnow)
+    filters = db.Column(db.PickleType, default=[])
+    private = db.Column(db.Boolean(), default=False)
+
+    # cache handling
+    etag = db.Column(db.String(), default="")
+    last_modified = db.Column(db.String(), default="")
+    last_retrieved = db.Column(db.DateTime(), default=datetime(1970, 1, 1))
+
+    # error logging
+    last_error = db.Column(db.String(), default="")
+    error_count = db.Column(db.Integer(), default=0)
+
+    # relationship
+    icon_url = db.Column(db.String(), db.ForeignKey("icon.url"), default=None)
+    user_id = db.Column(db.Integer(), db.ForeignKey("user.id"))
+    category_id = db.Column(db.Integer(), db.ForeignKey("category.id"))
+    articles = db.relationship(
+        Article,
+        backref="source",
+        lazy="dynamic",
+        cascade="all,delete-orphan",
+        order_by=desc(Article.date),
+    )
+
+    # index
+    idx_feed_uid_cid = Index("user_id", "category_id")
+    idx_feed_uid = Index("user_id")
+
+    # api whitelists
+    @staticmethod
+    def _fields_base_write():
+        return {
+            "title",
+            "description",
+            "link",
+            "site_link",
+            "enabled",
+            "filters",
+            "last_error",
+            "error_count",
+            "category_id",
+        }
+
+    @staticmethod
+    def _fields_base_read():
+        return {"id", "user_id", "icon_url", "last_retrieved"}
+
+    @validates("title")
+    def validates_title(self, key, value):
+        return str(value).strip()
+
+    @validates("description")
+    def validates_description(self, key, value):
+        return str(value).strip()
+
+    def __repr__(self):
+        return "<Feed %r>" % (self.title)
diff --git a/newspipe/models/icon.py b/newspipe/models/icon.py
new file mode 100644
index 00000000..99c10224
--- /dev/null
+++ b/newspipe/models/icon.py
@@ -0,0 +1,10 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from newspipe.bootstrap import db
+
+
+class Icon(db.Model):
+    url = db.Column(db.String(), primary_key=True)
+    content = db.Column(db.String(), default=None)
+    mimetype = db.Column(db.String(), default="application/image")
diff --git a/newspipe/models/right_mixin.py b/newspipe/models/right_mixin.py
new file mode 100644
index 00000000..670beafa
--- /dev/null
+++ b/newspipe/models/right_mixin.py
@@ -0,0 +1,64 @@
+from sqlalchemy.ext.associationproxy import _AssociationList
+
+
+class RightMixin:
+    @staticmethod
+    def _fields_base_write():
+        return set()
+
+    @staticmethod
+    def _fields_base_read():
+        return set(["id"])
+
+    @staticmethod
+    def _fields_api_write():
+        return set([])
+
+    @staticmethod
+    def _fields_api_read():
+        return set(["id"])
+
+    @classmethod
+    def fields_base_write(cls):
+        return cls._fields_base_write()
+
+    @classmethod
+    def fields_base_read(cls):
+        return cls._fields_base_write().union(cls._fields_base_read())
+
+    @classmethod
+    def fields_api_write(cls):
+        return cls.fields_base_write().union(cls._fields_api_write())
+
+    @classmethod
+    def fields_api_read(cls):
+        return cls.fields_base_read().union(cls._fields_api_read())
+
+    def __getitem__(self, key):
+        if not hasattr(self, "__dump__"):
+            self.__dump__ = {}
+        return self.__dump__.get(key)
+
+    def __setitem__(self, key, value):
+        if not hasattr(self, "__dump__"):
+            self.__dump__ = {}
+        self.__dump__[key] = value
+
+    def dump(self, role="admin"):
+        if role == "admin":
+            dico = {
+                k: getattr(self, k)
+                for k in set(self.__table__.columns.keys())
+                .union(self.fields_api_read())
+                .union(self.fields_base_read())
+            }
+        elif role == "api":
+            dico = {k: getattr(self, k) for k in self.fields_api_read()}
+        else:
+            dico = {k: getattr(self, k) for k in self.fields_base_read()}
+        if hasattr(self, "__dump__"):
+            dico.update(self.__dump__)
+        for key, value in dico.items():  # preventing association proxy to die
+            if isinstance(value, _AssociationList):
+                dico[key] = list(value)
+        return dico
diff --git a/newspipe/models/role.py b/newspipe/models/role.py
new file mode 100644
index 00000000..628467c0
--- /dev/null
+++ b/newspipe/models/role.py
@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from newspipe.bootstrap import db
+
+
+class Role(db.Model):
+    """
+    Represent a role.
+    """
+
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(), unique=True)
+
+    user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
diff --git a/newspipe/models/tag.py b/newspipe/models/tag.py
new file mode 100644
index 00000000..b853c344
--- /dev/null
+++ b/newspipe/models/tag.py
@@ -0,0 +1,44 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from newspipe.bootstrap import db
+
+
+class ArticleTag(db.Model):
+    text = db.Column(db.String, primary_key=True, unique=False)
+
+    # foreign keys
+    article_id = db.Column(
+        db.Integer, db.ForeignKey("article.id", ondelete="CASCADE"), primary_key=True
+    )
+
+    # relationships
+    article = db.relationship(
+        "Article", back_populates="tag_objs", foreign_keys=[article_id]
+    )
+
+    def __init__(self, text):
+        self.text = text
+
+
+class BookmarkTag(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    text = db.Column(db.String, unique=False)
+
+    # foreign keys
+    user_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE"))
+    bookmark_id = db.Column(
+        db.Integer, db.ForeignKey("bookmark.id", ondelete="CASCADE")
+    )
+
+    # relationships
+    bookmark = db.relationship(
+        "Bookmark",
+        back_populates="tags",
+        cascade="all,delete",
+        foreign_keys=[bookmark_id],
+    )
+
+    # def __init__(self, text, user_id):
+    #     self.text = text
+    #     self.user_id = user_id
diff --git a/newspipe/models/user.py b/newspipe/models/user.py
new file mode 100644
index 00000000..142536fe
--- /dev/null
+++ b/newspipe/models/user.py
@@ -0,0 +1,114 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# newspipe - A Web based news aggregator.
+# Copyright (C) 2010-2020 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information: https://git.sr.ht/~cedric/newspipe
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+import re
+import random
+import hashlib
+from datetime import datetime
+from werkzeug.security import check_password_hash
+from flask_login import UserMixin
+from sqlalchemy.orm import validates
+
+from newspipe.bootstrap import db
+from newspipe.models.right_mixin import RightMixin
+from newspipe.models.category import Category
+from newspipe.models.feed import Feed
+
+
+class User(db.Model, UserMixin, RightMixin):
+    """
+    Represent a user.
+    """
+
+    id = db.Column(db.Integer, primary_key=True)
+    nickname = db.Column(db.String(), unique=True)
+    pwdhash = db.Column(db.String())
+
+    automatic_crawling = db.Column(db.Boolean(), default=True)
+
+    is_public_profile = db.Column(db.Boolean(), default=False)
+    bio = db.Column(db.String(5000), default="")
+    webpage = db.Column(db.String(), default="")
+    twitter = db.Column(db.String(), default="")
+
+    date_created = db.Column(db.DateTime(), default=datetime.utcnow)
+    last_seen = db.Column(db.DateTime(), default=datetime.utcnow)
+
+    # user rights
+    is_active = db.Column(db.Boolean(), default=False)
+    is_admin = db.Column(db.Boolean(), default=False)
+    is_api = db.Column(db.Boolean(), default=False)
+
+    # relationships
+    categories = db.relationship(
+        "Category",
+        backref="user",
+        cascade="all, delete-orphan",
+        foreign_keys=[Category.user_id],
+    )
+    feeds = db.relationship(
+        "Feed",
+        backref="user",
+        cascade="all, delete-orphan",
+        foreign_keys=[Feed.user_id],
+    )
+
+    @staticmethod
+    def _fields_base_write():
+        return {"login", "password"}
+
+    @staticmethod
+    def _fields_base_read():
+        return {"date_created", "last_connection"}
+
+    @staticmethod
+    def make_valid_nickname(nickname):
+        return re.sub("[^a-zA-Z0-9_\.]", "", nickname)
+
+    @validates("bio")
+    def validates_bio(self, key, value):
+        assert len(value) <= 5000, AssertionError("maximum length for bio: 5000")
+        return value.strip()
+
+    def get_id(self):
+        """
+        Return the id of the user.
+        """
+        return self.id
+
+    def check_password(self, password):
+        """
+        Check the password of the user.
+        """
+        return check_password_hash(self.pwdhash, password)
+
+    def __eq__(self, other):
+        return self.id == other.id
+
+    def __repr__(self):
+        return "<User %r>" % (self.nickname)