From 2849c82255b4b889c7342a0a8fa8a4aecfbe599d Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Sat, 17 Jan 2015 16:50:38 +0100
Subject: a first big refacto of the existing arch

---
 pyaggr3g470r/controllers/__init__.py |  5 ++++
 pyaggr3g470r/controllers/abstract.py | 47 ++++++++++++++++++++++++++++++++++++
 pyaggr3g470r/controllers/article.py  | 20 +++++++++++++++
 pyaggr3g470r/controllers/feed.py     |  6 +++++
 4 files changed, 78 insertions(+)
 create mode 100644 pyaggr3g470r/controllers/__init__.py
 create mode 100644 pyaggr3g470r/controllers/abstract.py
 create mode 100644 pyaggr3g470r/controllers/article.py
 create mode 100644 pyaggr3g470r/controllers/feed.py

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py
new file mode 100644
index 00000000..029cbf43
--- /dev/null
+++ b/pyaggr3g470r/controllers/__init__.py
@@ -0,0 +1,5 @@
+from .feed import FeedController
+from .article import ArticleController
+
+
+__all__ = ['FeedController', 'ArticleController']
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
new file mode 100644
index 00000000..fe437b09
--- /dev/null
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -0,0 +1,47 @@
+from flask import g
+from pyaggr3g470r.lib.exceptions import Forbidden, NotFound
+
+
+class AbstractController(object):
+    _db_cls = None
+
+    def __init__(self, user_id):
+        self.user_id = user_id
+
+    def _get(self, **filters):
+        if self.user_id:
+            filters['user_id'] = self.user_id
+        db_filters = [getattr(self._db_cls, key) == value
+                      for key, value in filters.iteritems()]
+        return self._db_cls.query.filter(*db_filters).first()
+
+    def get(self, **filters):
+        obj = self._get(**filters).first()
+        if not obj:
+            raise NotFound({'message': 'No %r (%r)'
+                                % (self._db_cls.__class__.__name__, filters)})
+        if obj.user_id != self.user_id:
+            raise Forbidden({'message': 'No authorized to access %r (%r)'
+                                % (self._db_cls.__class__.__name__, filters)})
+        return obj
+
+    def create(self, **attrs):
+        obj = self._db_cls(**attrs)
+        g.db.session.commit()
+        return obj
+
+    def read(self, **filters):
+        return self._get(**filters)
+
+    def update(self, obj_id, **attrs):
+        obj = self.get(id=obj_id)
+        for key, values in attrs.iteritems():
+            setattr(obj, key, values)
+        g.db.session.commit()
+        return obj
+
+    def delete(self, obj_id):
+        obj = self.get(id=obj_id)
+        g.db.session.delete(obj)
+        g.db.session.commit()
+        return obj
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
new file mode 100644
index 00000000..cfaf386d
--- /dev/null
+++ b/pyaggr3g470r/controllers/article.py
@@ -0,0 +1,20 @@
+import conf
+from .abstract import AbstractController
+from pyaggr3g470r.models import Article
+
+
+class ArticleController(AbstractController):
+    _db_cls = Article
+
+    def read(self, obj_id):
+        article = super(ArticleController, self).read(obj_id)
+        if not article.readed:
+            self.update(obj_id, readed=True)
+        return article
+
+    def delete(self, obj_id):
+        obj = super(ArticleController, self).delete(obj_id)
+        if not conf.ON_HEROKU:
+            import pyaggr3g470r.search as fastsearch
+            fastsearch.delete_article(self.user_id, obj.feed_id, obj_id)
+        return obj
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
new file mode 100644
index 00000000..5ada1a48
--- /dev/null
+++ b/pyaggr3g470r/controllers/feed.py
@@ -0,0 +1,6 @@
+from .abstract import AbstractController
+from pyaggr3g470r.models import Feed
+
+
+class FeedController(AbstractController):
+    _db_cls = Feed
-- 
cgit 


From 5ce0ce0d57c9d9976a47a120ca6235b84ade236a Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Tue, 20 Jan 2015 17:20:07 +0100
Subject: first implementation of fetchable feeds

---
 pyaggr3g470r/controllers/__init__.py |  3 ++-
 pyaggr3g470r/controllers/abstract.py | 43 +++++++++++++++++++++++-------------
 pyaggr3g470r/controllers/feed.py     | 17 ++++++++++++++
 pyaggr3g470r/controllers/user.py     |  7 ++++++
 4 files changed, 54 insertions(+), 16 deletions(-)
 create mode 100644 pyaggr3g470r/controllers/user.py

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py
index 029cbf43..d8d1a104 100644
--- a/pyaggr3g470r/controllers/__init__.py
+++ b/pyaggr3g470r/controllers/__init__.py
@@ -1,5 +1,6 @@
 from .feed import FeedController
 from .article import ArticleController
+from .user import UserController
 
 
-__all__ = ['FeedController', 'ArticleController']
+__all__ = ['FeedController', 'ArticleController', 'UserController']
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
index fe437b09..8960c3be 100644
--- a/pyaggr3g470r/controllers/abstract.py
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -1,47 +1,60 @@
-from flask import g
+from bootstrap import db
 from pyaggr3g470r.lib.exceptions import Forbidden, NotFound
 
 
 class AbstractController(object):
     _db_cls = None
+    _user_id_key = 'user_id'
 
     def __init__(self, user_id):
         self.user_id = user_id
 
     def _get(self, **filters):
         if self.user_id:
-            filters['user_id'] = self.user_id
-        db_filters = [getattr(self._db_cls, key) == value
-                      for key, value in filters.iteritems()]
-        return self._db_cls.query.filter(*db_filters).first()
+            filters[self._user_id_key] = self.user_id
+        db_filters = set()
+        for key, value in filters.iteritems():
+            if key.endswith('__gt'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) > value)
+            elif key.endswith('__lt'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) < value)
+            elif key.endswith('__ge'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) >= value)
+            elif key.endswith('__le'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) <= value)
+            elif key.endswith('__ne'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) != value)
+            elif key.endswith('__in'):
+                db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
+            else:
+                db_filters.add(getattr(self._db_cls, key) == value)
+        return self._db_cls.query.filter(*db_filters)
 
     def get(self, **filters):
         obj = self._get(**filters).first()
         if not obj:
             raise NotFound({'message': 'No %r (%r)'
                                 % (self._db_cls.__class__.__name__, filters)})
-        if obj.user_id != self.user_id:
+        if getattr(obj, self._user_id_key) != self.user_id:
             raise Forbidden({'message': 'No authorized to access %r (%r)'
                                 % (self._db_cls.__class__.__name__, filters)})
         return obj
 
     def create(self, **attrs):
         obj = self._db_cls(**attrs)
-        g.db.session.commit()
+        db.session.commit()
         return obj
 
     def read(self, **filters):
         return self._get(**filters)
 
-    def update(self, obj_id, **attrs):
-        obj = self.get(id=obj_id)
-        for key, values in attrs.iteritems():
-            setattr(obj, key, values)
-        g.db.session.commit()
-        return obj
+    def update(self, filters, attrs):
+        result = self._get(**filters).update(attrs, synchronize_session=False)
+        db.session.commit()
+        return result
 
     def delete(self, obj_id):
         obj = self.get(id=obj_id)
-        g.db.session.delete(obj)
-        g.db.session.commit()
+        db.session.delete(obj)
+        db.session.commit()
         return obj
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index 5ada1a48..ce1c413f 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -1,6 +1,23 @@
+from datetime import datetime, timedelta
 from .abstract import AbstractController
 from pyaggr3g470r.models import Feed
 
+DEFAULT_MAX_ERROR = 3
+DEFAULT_LIMIT = 5
+
 
 class FeedController(AbstractController):
     _db_cls = Feed
+
+    def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
+        from pyaggr3g470r.controllers import UserController
+        now = datetime.now()
+        user = UserController(self.user_id).get(id=self.user_id)
+        max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60)
+        feeds = [feed for feed in self.read(user_id=self.user_id,
+                          error_count__le=max_error,
+                          last_refreshed__lt=max_last_refresh).limit(limit)]
+
+        self.update({'id__in': [feed.id for feed in feeds]},
+                    {'last_refreshed': now})
+        return feeds
diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py
new file mode 100644
index 00000000..c6c1d545
--- /dev/null
+++ b/pyaggr3g470r/controllers/user.py
@@ -0,0 +1,7 @@
+from .abstract import AbstractController
+from pyaggr3g470r.models import User
+
+
+class UserController(AbstractController):
+    _db_cls = User
+    _user_id_key = 'id'
-- 
cgit 


From 4f0ad9e442e64f69d420dea4d737805eefaaf981 Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Wed, 21 Jan 2015 14:07:00 +0100
Subject: continuing refacto

---
 pyaggr3g470r/controllers/abstract.py |  2 +-
 pyaggr3g470r/controllers/article.py  |  6 +++---
 pyaggr3g470r/controllers/feed.py     | 10 ++++++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
index 8960c3be..6fe45461 100644
--- a/pyaggr3g470r/controllers/abstract.py
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -13,7 +13,7 @@ class AbstractController(object):
         if self.user_id:
             filters[self._user_id_key] = self.user_id
         db_filters = set()
-        for key, value in filters.iteritems():
+        for key, value in filters.items():
             if key.endswith('__gt'):
                 db_filters.add(getattr(self._db_cls, key[:-4]) > value)
             elif key.endswith('__lt'):
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
index cfaf386d..0de223ee 100644
--- a/pyaggr3g470r/controllers/article.py
+++ b/pyaggr3g470r/controllers/article.py
@@ -6,10 +6,10 @@ from pyaggr3g470r.models import Article
 class ArticleController(AbstractController):
     _db_cls = Article
 
-    def read(self, obj_id):
-        article = super(ArticleController, self).read(obj_id)
+    def get(self, **filters):
+        article = super(ArticleController, self).read(**filters)
         if not article.readed:
-            self.update(obj_id, readed=True)
+            self.update(article.id, readed=True)
         return article
 
     def delete(self, obj_id):
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index ce1c413f..56cef997 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -16,8 +16,14 @@ class FeedController(AbstractController):
         max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60)
         feeds = [feed for feed in self.read(user_id=self.user_id,
                           error_count__le=max_error,
-                          last_refreshed__lt=max_last_refresh).limit(limit)]
+                          last_modified=max_last_refresh).limit(limit)]
 
         self.update({'id__in': [feed.id for feed in feeds]},
-                    {'last_refreshed': now})
+                    {'last_modified': now})
         return feeds
+
+    def list_last_articles(self, feed_id, limit=50):
+        from pyaggr3g470r.controllers import ArticleController
+        return ArticleController(self.user_id)._get(feed_id=feed_id)\
+                .order_by(ArticleController._db_cls.retrieved_date.desc())\
+                .limit(limit)
-- 
cgit 


From 5572851eca3b2f1bc56aed7232284acc436d2f49 Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Sun, 1 Mar 2015 03:20:12 +0100
Subject: new crawler with cache control and error handling

---
 pyaggr3g470r/controllers/abstract.py | 15 ++++++++++++---
 pyaggr3g470r/controllers/article.py  | 11 +++++++++--
 pyaggr3g470r/controllers/feed.py     | 17 ++++++-----------
 3 files changed, 27 insertions(+), 16 deletions(-)

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
index 6fe45461..ebb73e30 100644
--- a/pyaggr3g470r/controllers/abstract.py
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -1,5 +1,9 @@
+import logging
 from bootstrap import db
-from pyaggr3g470r.lib.exceptions import Forbidden, NotFound
+from sqlalchemy import update
+from werkzeug.exceptions import Forbidden, NotFound
+
+logger = logging.getLogger(__name__)
 
 
 class AbstractController(object):
@@ -9,7 +13,7 @@ class AbstractController(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
-    def _get(self, **filters):
+    def _to_filters(self, **filters):
         if self.user_id:
             filters[self._user_id_key] = self.user_id
         db_filters = set()
@@ -28,7 +32,10 @@ class AbstractController(object):
                 db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
             else:
                 db_filters.add(getattr(self._db_cls, key) == value)
-        return self._db_cls.query.filter(*db_filters)
+        return db_filters
+
+    def _get(self, **filters):
+        return self._db_cls.query.filter(*self._to_filters(**filters))
 
     def get(self, **filters):
         obj = self._get(**filters).first()
@@ -41,7 +48,9 @@ class AbstractController(object):
         return obj
 
     def create(self, **attrs):
+        attrs['user_id'] = self.user_id
         obj = self._db_cls(**attrs)
+        db.session.add(obj)
         db.session.commit()
         return obj
 
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
index 0de223ee..46ca0988 100644
--- a/pyaggr3g470r/controllers/article.py
+++ b/pyaggr3g470r/controllers/article.py
@@ -7,9 +7,9 @@ class ArticleController(AbstractController):
     _db_cls = Article
 
     def get(self, **filters):
-        article = super(ArticleController, self).read(**filters)
+        article = super(ArticleController, self).get(**filters)
         if not article.readed:
-            self.update(article.id, readed=True)
+            self.update({'id': article.id}, {'readed': True})
         return article
 
     def delete(self, obj_id):
@@ -18,3 +18,10 @@ class ArticleController(AbstractController):
             import pyaggr3g470r.search as fastsearch
             fastsearch.delete_article(self.user_id, obj.feed_id, obj_id)
         return obj
+
+    def challenge(self, ids):
+        """Will return each id that wasn't found in the database."""
+        for id_ in ids:
+            if self.read(**id_).first():
+                continue
+            yield id_
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index 56cef997..286bea1e 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -13,17 +13,12 @@ class FeedController(AbstractController):
         from pyaggr3g470r.controllers import UserController
         now = datetime.now()
         user = UserController(self.user_id).get(id=self.user_id)
-        max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60)
+        max_last = now - timedelta(minutes=user.refresh_rate or 60)
         feeds = [feed for feed in self.read(user_id=self.user_id,
-                          error_count__le=max_error,
-                          last_modified=max_last_refresh).limit(limit)]
+                            error_count__le=max_error, enabled=True,
+                            last_retreived__lt=max_last).limit(limit)]
 
-        self.update({'id__in': [feed.id for feed in feeds]},
-                    {'last_modified': now})
+        if feeds:
+            self.update({'id__in': [feed.id for feed in feeds]},
+                        {'last_retreived': now})
         return feeds
-
-    def list_last_articles(self, feed_id, limit=50):
-        from pyaggr3g470r.controllers import ArticleController
-        return ArticleController(self.user_id)._get(feed_id=feed_id)\
-                .order_by(ArticleController._db_cls.retrieved_date.desc())\
-                .limit(limit)
-- 
cgit 


From 643f4590445928b7ac568b922f1edb6f52765b68 Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Tue, 3 Mar 2015 00:02:09 +0100
Subject: displaying feed errors in ui

---
 pyaggr3g470r/controllers/feed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index 286bea1e..a2455e2b 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -15,7 +15,7 @@ class FeedController(AbstractController):
         user = UserController(self.user_id).get(id=self.user_id)
         max_last = now - timedelta(minutes=user.refresh_rate or 60)
         feeds = [feed for feed in self.read(user_id=self.user_id,
-                            error_count__le=max_error, enabled=True,
+                            error_count__lt=max_error, enabled=True,
                             last_retreived__lt=max_last).limit(limit)]
 
         if feeds:
-- 
cgit 


From 631fc8a3ebaf74dc609a445dc0b11b73eb0eab02 Mon Sep 17 00:00:00 2001
From: François Schmidts <francois.schmidts@gmail.com>
Date: Tue, 3 Mar 2015 18:12:11 +0100
Subject: adding some docstring

---
 pyaggr3g470r/controllers/abstract.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'pyaggr3g470r/controllers')

diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
index ebb73e30..a99e67f3 100644
--- a/pyaggr3g470r/controllers/abstract.py
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 
 class AbstractController(object):
-    _db_cls = None
+    _db_cls = None  # reference to the database class
     _user_id_key = 'user_id'
 
     def __init__(self, user_id):
@@ -48,7 +48,7 @@ class AbstractController(object):
         return obj
 
     def create(self, **attrs):
-        attrs['user_id'] = self.user_id
+        attrs[self._user_id_key] = self.user_id
         obj = self._db_cls(**attrs)
         db.session.add(obj)
         db.session.commit()
-- 
cgit