aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/lib/crawler.py
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-08-03 23:55:46 +0200
committerCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-08-03 23:55:46 +0200
commitcdbd573500a365e290e88b50d7b0c2355b7f7e19 (patch)
tree25ede52ae4b02a2377ae40d2c146c7ed2e9abe2a /pyaggr3g470r/lib/crawler.py
parentThe numver ov values of the splited string is variable (sometimes the charset... (diff)
parentsqlalchemy was requesting icons everytime feed where listed (diff)
downloadnewspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.tar.gz
newspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.tar.bz2
newspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.zip
Merged in jaesivsm/pyaggr3g470r (pull request #20)
perf improvement
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
-rw-r--r--pyaggr3g470r/lib/crawler.py9
1 files changed, 5 insertions, 4 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index e5998776..216e7a96 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -125,7 +125,7 @@ class PyAggUpdater(AbstractCrawler):
entry = construct_article(
self.entries[tuple(sorted(id_to_create.items()))],
self.feed)
- logger.warn('%r %r - creating %r for %r - %r', self.feed['id'],
+ logger.info('%r %r - creating %r for %r - %r', self.feed['id'],
self.feed['title'], entry['title'], entry['user_id'],
id_to_create)
self.query_pyagg('post', 'article', entry)
@@ -141,7 +141,7 @@ class PyAggUpdater(AbstractCrawler):
strftime('%a, %d %b %Y %X %Z', gmtime()))}
fresh_feed = construct_feed_from(url=self.feed['link'],
fp_parsed=self.parsed_feed)
- for key in ('description', 'site_link', 'icon'):
+ for key in ('description', 'site_link', 'icon_url'):
if fresh_feed.get(key) and fresh_feed[key] != self.feed.get(key):
up_feed[key] = fresh_feed[key]
if not self.feed.get('title'):
@@ -152,11 +152,12 @@ class PyAggUpdater(AbstractCrawler):
up_feed['last_retrieved'] \
= (datetime.now() - timedelta(minutes=45)).isoformat()
- logger.info('%r %r - pushing feed attrs %r',
+ if any([up_feed[key] != self.feed.get(key) for key in up_feed]):
+ logger.warn('%r %r - pushing feed attrs %r',
self.feed['id'], self.feed['title'],
{key: "%s -> %s" % (up_feed[key], self.feed.get(key))
for key in up_feed if up_feed[key] != self.feed.get(key)})
- if any([up_feed[key] != self.feed.get(key) for key in up_feed]):
+
future = self.query_pyagg('put',
'feed/%d' % self.feed['id'], up_feed)
future.add_done_callback(self.get_counter_callback())
bgstack15