diff options
| author | François Schmidts <francois.schmidts@gmail.com> | 2015-03-03 00:23:34 +0100 |
|---|---|---|
| committer | François Schmidts <francois.schmidts@gmail.com> | 2015-03-03 22:23:47 +0100 |
| commit | cafcd09ff278aa775adafce936a54ce4d2aec5fd (patch) | |
| tree | 007afadbeaf52c16631da9c2c63c3ec9d1ab4c1b /pyaggr3g470r/lib/crawler.py | |
| parent | displaying feed errors in ui (diff) | |
| download | newspipe-cafcd09ff278aa775adafce936a54ce4d2aec5fd.tar.gz newspipe-cafcd09ff278aa775adafce936a54ce4d2aec5fd.tar.bz2 newspipe-cafcd09ff278aa775adafce936a54ce4d2aec5fd.zip | |
cleaning feed on okay download resulting in 304
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 5e828dbf..1b9f5d60 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -139,6 +139,11 @@ class FeedCrawler(AbstractCrawler): self.feed = feed super(FeedCrawler, self).__init__(auth) + def clean_feed(self): + if self.feed.get('error_count') or self.feed.get('last_error'): + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': 0, 'last_error': ''}) + @AbstractCrawler.count_on_me def callback(self, response): try: @@ -157,12 +162,14 @@ class FeedCrawler(AbstractCrawler): if response.status_code == 304: logger.info("%r %r - feed responded with 304", self.feed['id'], self.feed['title']) + self.clean_feed() return if self.feed['etag'] and response.headers.get('etag') \ and response.headers.get('etag') == self.feed['etag']: logger.info("%r %r - feed responded with same etag (%d)", self.feed['id'], self.feed['title'], response.status_code) + self.clean_feed() return ids, entries = [], {} parsed_response = feedparser.parse(response.text) |