diff options
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 5e828dbf..1b9f5d60 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -139,6 +139,11 @@ class FeedCrawler(AbstractCrawler): self.feed = feed super(FeedCrawler, self).__init__(auth) + def clean_feed(self): + if self.feed.get('error_count') or self.feed.get('last_error'): + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': 0, 'last_error': ''}) + @AbstractCrawler.count_on_me def callback(self, response): try: @@ -157,12 +162,14 @@ class FeedCrawler(AbstractCrawler): if response.status_code == 304: logger.info("%r %r - feed responded with 304", self.feed['id'], self.feed['title']) + self.clean_feed() return if self.feed['etag'] and response.headers.get('etag') \ and response.headers.get('etag') == self.feed['etag']: logger.info("%r %r - feed responded with same etag (%d)", self.feed['id'], self.feed['title'], response.status_code) + self.clean_feed() return ids, entries = [], {} parsed_response = feedparser.parse(response.text) |