diff options
| -rw-r--r-- | pyaggr3g470r/crawler.py | 17 |
1 files changed, 10 insertions, 7 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index e153c1ab..cada119b 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -71,8 +71,6 @@ def parse_feed(user, feed): finally: if data is None: feed.error_count += 1 - if feed.error_count >= conf.DEFAULT_MAX_ERROR: - feed.enabled = False db.session.commit() return @@ -81,8 +79,6 @@ def parse_feed(user, feed): #logger.error(a_feed['bozo_exception']) feed.last_error = str(a_feed['bozo_exception']) feed.error_count += 1 - if feed.error_count >= conf.DEFAULT_MAX_ERROR: - feed.enabled = False db.session.commit() if a_feed['entries'] == []: return @@ -235,7 +231,9 @@ def retrieve_feed(user, feed_id=None): # Get the list of feeds to fetch user = User.query.filter(User.email == user.email).first() - feeds = [feed for feed in user.feeds if feed.enabled] + feeds = [feed for feed in user.feeds if + feed.error_count <= conf.DEFAULT_MAX_ERROR and \ + feed.enabled] if feed_id is not None: feeds = [feed for feed in feeds if feed.id == feed_id] @@ -244,8 +242,13 @@ def retrieve_feed(user, feed_id=None): # Launch the process for all the feeds loop = asyncio.get_event_loop() - tasks = [init_process(user, feed) for feed in feeds] - #tasks = [asyncio.async(init_process(user, feed)) for feed in feeds] + tasks = [] + try: + # Python 3.5 (test) + tasks = [asyncio.ensure_future(init_process(user, feed)) for feed in feeds] + except: + tasks = [init_process(user, feed) for feed in feeds] loop.run_until_complete(asyncio.wait(tasks)) + loop.close() logger.info("All articles retrieved. End of the processus.") |