aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/crawler.py
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-04-06 10:37:13 +0200
committerFrançois Schmidts <francois.schmidts@gmail.com>2015-04-06 10:37:13 +0200
commit29bb2a36f0b5d1781ab05f1976aa0c5017351807 (patch)
tree52b2dd87f4d36f6a9c518cc14f96e523b1dea045 /pyaggr3g470r/crawler.py
parentmisc update (diff)
parentMinor changes to the CSS. (diff)
downloadnewspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.tar.gz
newspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.tar.bz2
newspipe-29bb2a36f0b5d1781ab05f1976aa0c5017351807.zip
Merge remote-tracking branch 'upstream/master'
Conflicts: pyaggr3g470r/controllers/feed.py pyaggr3g470r/templates/home.html
Diffstat (limited to 'pyaggr3g470r/crawler.py')
-rw-r--r--pyaggr3g470r/crawler.py22
1 files changed, 13 insertions, 9 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index ebcb8ce4..ded9df6f 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -56,7 +56,7 @@ def get(*args, **kwargs):
return (yield from response.read_and_close(decode=False))
except Exception as e:
#print(e)
- return None
+ raise e
@asyncio.coroutine
def parse_feed(user, feed):
@@ -66,14 +66,17 @@ def parse_feed(user, feed):
data = None
with (yield from sem):
- data = yield from get(feed.link)
-
- if data is None:
- feed.error_count += 1
- if feed.error_count > 2:
- feed.enabled = False
- db.session.commit()
- return
+ try:
+ data = yield from get(feed.link)
+ except Exception as e:
+ feed.last_error = str(e)
+ finally:
+ if data is None:
+ feed.error_count += 1
+ if feed.error_count > 2:
+ feed.enabled = False
+ db.session.commit()
+ return
a_feed = feedparser.parse(data)
if a_feed['bozo'] == 1:
@@ -88,6 +91,7 @@ def parse_feed(user, feed):
feed.last_retrieved = datetime.now(dateutil.tz.tzlocal())
feed.error_count = 0
+ feed.last_error = ""
# Feed informations
if feed.title == "":
bgstack15