aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pyaggr3g470r/lib/crawler.py29
-rw-r--r--pyaggr3g470r/views/api/article.py3
-rw-r--r--pyaggr3g470r/views/api/feed.py3
3 files changed, 21 insertions, 14 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index 216e7a96..cca3245e 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -116,19 +116,21 @@ class PyAggUpdater(AbstractCrawler):
"""Will process the result from the challenge, creating missing article
and updating the feed"""
AbstractCrawler.__counter__ -= 1
- results = response.result().json()
- logger.debug('%r %r - %d entries were not matched and will be created',
- self.feed['id'], self.feed['title'], len(results))
article_created = False
- for id_to_create in results:
- article_created = True
- entry = construct_article(
- self.entries[tuple(sorted(id_to_create.items()))],
- self.feed)
- logger.info('%r %r - creating %r for %r - %r', self.feed['id'],
- self.feed['title'], entry['title'], entry['user_id'],
- id_to_create)
- self.query_pyagg('post', 'article', entry)
+ if response.result().status_code != 204:
+ results = response.result().json()
+ logger.debug('%r %r - %d entries were not matched '
+ 'and will be created',
+ self.feed['id'], self.feed['title'], len(results))
+ for id_to_create in results:
+ article_created = True
+ entry = construct_article(
+ self.entries[tuple(sorted(id_to_create.items()))],
+ self.feed)
+ logger.info('%r %r - creating %r for %r - %r', self.feed['id'],
+ self.feed['title'], entry['title'],
+ entry['user_id'], id_to_create)
+ self.query_pyagg('post', 'article', entry)
logger.debug('%r %r - updating feed etag %r last_mod %r',
self.feed['id'], self.feed['title'],
@@ -263,6 +265,9 @@ class CrawlerScheduler(AbstractCrawler):
AbstractCrawler.__counter__ -= 1
response = response.result()
response.raise_for_status()
+ if response.status_code == 204:
+ logger.debug("No feed to fetch")
+ return
feeds = response.json()
logger.debug('%d to fetch %r', len(feeds), feeds)
for feed in feeds:
diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py
index 03ecdb18..d2969cb0 100644
--- a/pyaggr3g470r/views/api/article.py
+++ b/pyaggr3g470r/views/api/article.py
@@ -51,7 +51,8 @@ class ArticlesChallenge(PyAggAbstractResource):
if key in id_dict:
id_dict[key] = dateutil.parser.parse(id_dict[key])
- return self.wider_controller.challenge(parsed_args['ids'])
+ result = list(self.wider_controller.challenge(parsed_args['ids']))
+ return result or None, 200 if result else 204
g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
index 7d8cdf38..c80e9a9b 100644
--- a/pyaggr3g470r/views/api/feed.py
+++ b/pyaggr3g470r/views/api/feed.py
@@ -58,7 +58,8 @@ class FetchableFeedAPI(PyAggAbstractResource):
contr = self.wider_controller
else:
contr = self.controller
- return [feed for feed in contr.list_fetchable(**args)]
+ result = [feed for feed in contr.list_fetchable(**args)]
+ return result or None, 200 if result else 204
g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')
bgstack15