From 822e59f043ba7b12962c5e65f59f2fd33a339f54 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Fri, 6 Mar 2015 11:07:43 +0100 Subject: better crawling crontrol --- pyaggr3g470r/controllers/feed.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'pyaggr3g470r/controllers/feed.py') diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index b8e28ee6..ff496efc 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -1,10 +1,10 @@ +import logging from datetime import datetime, timedelta + from .abstract import AbstractController from pyaggr3g470r.models import Feed -import logging logger = logging.getLogger(__name__) - DEFAULT_MAX_ERROR = 3 DEFAULT_LIMIT = 5 @@ -12,14 +12,19 @@ DEFAULT_LIMIT = 5 class FeedController(AbstractController): _db_cls = Feed + def list_late(self, max_last, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + return [feed for feed in self.read( + error_count__lt=max_error, enabled=True, + last_retrieved__lt=max_last) + .order_by('Feed.last_retrieved') + .limit(limit)] + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): from pyaggr3g470r.controllers import UserController now = datetime.now() user = UserController(self.user_id).get(id=self.user_id) max_last = now - timedelta(minutes=user.refresh_rate or 60) - feeds = [feed for feed in self.read(user_id=self.user_id, - error_count__lt=max_error, enabled=True, - last_retrieved__lt=max_last).limit(limit)] + feeds = self.list_late(max_last, max_error, limit) if feeds: self.update({'id__in': [feed.id for feed in feeds]}, {'last_retrieved': now}) -- cgit