diff options
author | Cédric Bonhomme <kimble.mandel+bitbucket@gmail.com> | 2015-07-10 14:10:34 +0200 |
---|---|---|
committer | Cédric Bonhomme <kimble.mandel+bitbucket@gmail.com> | 2015-07-10 14:10:34 +0200 |
commit | 3cb10b3a240d0419338f4f8df26c684ef3ebdf92 (patch) | |
tree | 12b8cc24e0ef68c95c6950e306c4494ed3c1f273 /pyaggr3g470r/lib/crawler.py | |
parent | Merge branch 'master' of bitbucket.org:cedricbonhomme/pyaggr3g470r (diff) | |
parent | rebuilding feed url as well (diff) | |
download | newspipe-3cb10b3a240d0419338f4f8df26c684ef3ebdf92.tar.gz newspipe-3cb10b3a240d0419338f4f8df26c684ef3ebdf92.tar.bz2 newspipe-3cb10b3a240d0419338f4f8df26c684ef3ebdf92.zip |
Merged in jaesivsm/pyaggr3g470r (pull request #18)
bookmarlet & navbar form improvement
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
-rw-r--r-- | pyaggr3g470r/lib/crawler.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index de557e45..e5998776 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -17,6 +17,7 @@ import conf import json import logging import feedparser +from datetime import datetime, timedelta from functools import wraps from time import strftime, gmtime from concurrent.futures import ThreadPoolExecutor @@ -118,7 +119,9 @@ class PyAggUpdater(AbstractCrawler): results = response.result().json() logger.debug('%r %r - %d entries were not matched and will be created', self.feed['id'], self.feed['title'], len(results)) + article_created = False for id_to_create in results: + article_created = True entry = construct_article( self.entries[tuple(sorted(id_to_create.items()))], self.feed) @@ -144,6 +147,10 @@ class PyAggUpdater(AbstractCrawler): if not self.feed.get('title'): up_feed['title'] = fresh_feed.get('title', '') up_feed['user_id'] = self.feed['user_id'] + # re-getting that feed earlier since new entries appeared + if article_created: + up_feed['last_retrieved'] \ + = (datetime.now() - timedelta(minutes=45)).isoformat() logger.info('%r %r - pushing feed attrs %r', self.feed['id'], self.feed['title'], |