From c9f812335a6bf89d19a78940b02c39b22c0af4b5 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Fri, 28 Oct 2016 08:07:39 +0200 Subject: a mess --- src/crawler/classic_crawler.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py index fcbbe22b..4125dad7 100644 --- a/src/crawler/classic_crawler.py +++ b/src/crawler/classic_crawler.py @@ -79,7 +79,6 @@ async def parse_feed(user, feed): except Exception as e: up_feed['last_error'] = str(e) up_feed['error_count'] = feed.error_count + 1 - print(up_feed['error_count']) finally: up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal()) if parsed_feed is None: @@ -123,7 +122,7 @@ async def insert_database(user, feed): existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) except Exception as e: - print("existing_article_req: " + str(e)) + logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 @@ -132,15 +131,14 @@ async def insert_database(user, feed): # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) - print("Article %r (%r) already in the database.", - article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: new_updated_date = existing_article.date - print(e) + logger.exception("new_updated_date failed: " + str(e)) + if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace(tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ @@ -157,6 +155,7 @@ async def insert_database(user, feed): art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue + # insertion of the new article article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) -- cgit