diff options
| author | François Schmidts <francois.schmidts@gmail.com> | 2015-04-06 10:37:13 +0200 |
|---|---|---|
| committer | François Schmidts <francois.schmidts@gmail.com> | 2015-04-06 10:37:13 +0200 |
| commit | 29bb2a36f0b5d1781ab05f1976aa0c5017351807 (patch) | |
| tree | 52b2dd87f4d36f6a9c518cc14f96e523b1dea045 /pyaggr3g470r/crawler.py | |
| parent | misc update (diff) | |
| parent | Minor changes to the CSS. (diff) | |
| download | newspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.tar.gz newspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.tar.bz2 newspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.zip | |
Merge remote-tracking branch 'upstream/master'
Conflicts:
pyaggr3g470r/controllers/feed.py
pyaggr3g470r/templates/home.html
Diffstat (limited to 'pyaggr3g470r/crawler.py')
| -rw-r--r-- | pyaggr3g470r/crawler.py | 22 |
1 files changed, 13 insertions, 9 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index ebcb8ce4..ded9df6f 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -56,7 +56,7 @@ def get(*args, **kwargs): return (yield from response.read_and_close(decode=False)) except Exception as e: #print(e) - return None + raise e @asyncio.coroutine def parse_feed(user, feed): @@ -66,14 +66,17 @@ def parse_feed(user, feed): data = None with (yield from sem): - data = yield from get(feed.link) - - if data is None: - feed.error_count += 1 - if feed.error_count > 2: - feed.enabled = False - db.session.commit() - return + try: + data = yield from get(feed.link) + except Exception as e: + feed.last_error = str(e) + finally: + if data is None: + feed.error_count += 1 + if feed.error_count > 2: + feed.enabled = False + db.session.commit() + return a_feed = feedparser.parse(data) if a_feed['bozo'] == 1: @@ -88,6 +91,7 @@ def parse_feed(user, feed): feed.last_retrieved = datetime.now(dateutil.tz.tzlocal()) feed.error_count = 0 + feed.last_error = "" # Feed informations if feed.title == "": |