From 8c0c605ea0d59355ece2e4f2755acb535ab7b90d Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Thu, 5 Mar 2015 23:36:52 +0100 Subject: correcting wait counter and reactivating last_retrieved --- pyaggr3g470r/controllers/feed.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'pyaggr3g470r/controllers') diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index a3cdcddd..b8e28ee6 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -2,6 +2,9 @@ from datetime import datetime, timedelta from .abstract import AbstractController from pyaggr3g470r.models import Feed +import logging +logger = logging.getLogger(__name__) + DEFAULT_MAX_ERROR = 3 DEFAULT_LIMIT = 5 @@ -13,11 +16,11 @@ class FeedController(AbstractController): from pyaggr3g470r.controllers import UserController now = datetime.now() user = UserController(self.user_id).get(id=self.user_id) - #max_last = now - timedelta(minutes=user.refresh_rate or 60) + max_last = now - timedelta(minutes=user.refresh_rate or 60) feeds = [feed for feed in self.read(user_id=self.user_id, - error_count__lt=max_error, enabled=True).limit(limit)] - #last_retrieved__lt=max_last).limit(limit)] - """if feeds: + error_count__lt=max_error, enabled=True, + last_retrieved__lt=max_last).limit(limit)] + if feeds: self.update({'id__in': [feed.id for feed in feeds]}, - {'last_retrieved': now})""" + {'last_retrieved': now}) return feeds -- cgit From 822e59f043ba7b12962c5e65f59f2fd33a339f54 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Fri, 6 Mar 2015 11:07:43 +0100 Subject: better crawling crontrol --- pyaggr3g470r/controllers/feed.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'pyaggr3g470r/controllers') diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index b8e28ee6..ff496efc 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -1,10 +1,10 @@ +import logging from datetime import datetime, timedelta + from .abstract import AbstractController from pyaggr3g470r.models import Feed -import logging logger = logging.getLogger(__name__) - DEFAULT_MAX_ERROR = 3 DEFAULT_LIMIT = 5 @@ -12,14 +12,19 @@ DEFAULT_LIMIT = 5 class FeedController(AbstractController): _db_cls = Feed + def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + return [feed for feed in self.read( + error_count__lt=max_error, enabled=True, + last_retrieved__lt=max_last) + .order_by('Feed.last_retrieved') + .limit(limit)] + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): from pyaggr3g470r.controllers import UserController now = datetime.now() user = UserController(self.user_id).get(id=self.user_id) max_last = now - timedelta(minutes=user.refresh_rate or 60) - feeds = [feed for feed in self.read(user_id=self.user_id, - error_count__lt=max_error, enabled=True, - last_retrieved__lt=max_last).limit(limit)] + feeds = self.list_late(max_last, max_error, limit) if feeds: self.update({'id__in': [feed.id for feed in feeds]}, {'last_retrieved': now}) -- cgit From f2463bc333cc207ffa9ab935b7edf59a9894720d Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Mon, 6 Apr 2015 10:19:58 +0200 Subject: misc update updating the way we maintain feed up to date in the database fixing the counter bumping the minimum error count --- pyaggr3g470r/controllers/feed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'pyaggr3g470r/controllers') diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index ff496efc..b99a3a7f 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -5,14 +5,15 @@ from .abstract import AbstractController from pyaggr3g470r.models import Feed logger = logging.getLogger(__name__) -DEFAULT_MAX_ERROR = 3 +DEFAULT_MAX_ERROR = 6 DEFAULT_LIMIT = 5 class FeedController(AbstractController): _db_cls = Feed - def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, + limit=DEFAULT_LIMIT): return [feed for feed in self.read( error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last) -- cgit From e41348abe7bb3d336cc474ea1d246dc25390a104 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Mon, 6 Apr 2015 12:05:59 +0200 Subject: correcting the way we use the controllers and adding documentation --- pyaggr3g470r/controllers/abstract.py | 26 ++++++++++++++++++++++---- pyaggr3g470r/controllers/user.py | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) (limited to 'pyaggr3g470r/controllers') diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py index c084deb9..f1173817 100644 --- a/pyaggr3g470r/controllers/abstract.py +++ b/pyaggr3g470r/controllers/abstract.py @@ -9,11 +9,25 @@ class AbstractController(object): _db_cls = None # reference to the database class _user_id_key = 'user_id' - def __init__(self, user_id): + def __init__(self, user_id=None): + """User id is a right management mechanism that should be used to + filter objects in database on their denormalized "user_id" field + (or "id" field for users). + Should no user_id be provided, the Controller won't apply any filter + allowing for a kind of "super user" mode. + """ self.user_id = user_id def _to_filters(self, **filters): - if self.user_id: + """ + Will translate filters to sqlalchemy filter. + This method will also apply user_id restriction if available. + + each parameters of the function is treated as an equality unless the + name of the parameter ends with either "__gt", "__lt", "__ge", "__le", + "__ne" or "__in". + """ + if self.user_id is not None: filters[self._user_id_key] = self.user_id db_filters = set() for key, value in filters.items(): @@ -37,17 +51,21 @@ class AbstractController(object): return self._db_cls.query.filter(*self._to_filters(**filters)) def get(self, **filters): + """Will return one single objects corresponding to filters""" obj = self._get(**filters).first() if not obj: raise NotFound({'message': 'No %r (%r)' % (self._db_cls.__class__.__name__, filters)}) - if getattr(obj, self._user_id_key) != self.user_id: + if self.user_id is not None \ + and getattr(obj, self._user_id_key) != self.user_id: raise Forbidden({'message': 'No authorized to access %r (%r)' % (self._db_cls.__class__.__name__, filters)}) return obj def create(self, **attrs): - attrs[self._user_id_key] = self.user_id + assert self._user_id_key in attrs or self.user_id is not None, \ + "You must provide user_id one way or another" + attrs[self._user_id_key] = self.user_id or attrs.get(self._user_id_key) obj = self._db_cls(**attrs) db.session.add(obj) db.session.commit() diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py index ed46e1e7..c6c1d545 100644 --- a/pyaggr3g470r/controllers/user.py +++ b/pyaggr3g470r/controllers/user.py @@ -4,4 +4,4 @@ from pyaggr3g470r.models import User class UserController(AbstractController): _db_cls = User - _user_id_key = 'email' + _user_id_key = 'id' -- cgit