aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pyaggr3g470r/crawler.py17
1 files changed, 10 insertions, 7 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index e153c1ab..cada119b 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -71,8 +71,6 @@ def parse_feed(user, feed):
finally:
if data is None:
feed.error_count += 1
- if feed.error_count >= conf.DEFAULT_MAX_ERROR:
- feed.enabled = False
db.session.commit()
return
@@ -81,8 +79,6 @@ def parse_feed(user, feed):
#logger.error(a_feed['bozo_exception'])
feed.last_error = str(a_feed['bozo_exception'])
feed.error_count += 1
- if feed.error_count >= conf.DEFAULT_MAX_ERROR:
- feed.enabled = False
db.session.commit()
if a_feed['entries'] == []:
return
@@ -235,7 +231,9 @@ def retrieve_feed(user, feed_id=None):
# Get the list of feeds to fetch
user = User.query.filter(User.email == user.email).first()
- feeds = [feed for feed in user.feeds if feed.enabled]
+ feeds = [feed for feed in user.feeds if
+ feed.error_count <= conf.DEFAULT_MAX_ERROR and \
+ feed.enabled]
if feed_id is not None:
feeds = [feed for feed in feeds if feed.id == feed_id]
@@ -244,8 +242,13 @@ def retrieve_feed(user, feed_id=None):
# Launch the process for all the feeds
loop = asyncio.get_event_loop()
- tasks = [init_process(user, feed) for feed in feeds]
- #tasks = [asyncio.async(init_process(user, feed)) for feed in feeds]
+ tasks = []
+ try:
+ # Python 3.5 (test)
+ tasks = [asyncio.ensure_future(init_process(user, feed)) for feed in feeds]
+ except:
+ tasks = [init_process(user, feed) for feed in feeds]
loop.run_until_complete(asyncio.wait(tasks))
+ loop.close()
logger.info("All articles retrieved. End of the processus.")
bgstack15