diff options
| author | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2016-10-28 08:07:39 +0200 |
|---|---|---|
| committer | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2016-10-28 08:07:39 +0200 |
| commit | c9f812335a6bf89d19a78940b02c39b22c0af4b5 (patch) | |
| tree | 45baf34cb37fcc04f1e205082b6ae74d37ecf24a /src/crawler | |
| parent | test (diff) | |
| download | newspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.tar.gz newspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.tar.bz2 newspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.zip | |
a mess
Diffstat (limited to 'src/crawler')
| -rw-r--r-- | src/crawler/classic_crawler.py | 9 |
1 files changed, 4 insertions, 5 deletions
diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py index fcbbe22b..4125dad7 100644 --- a/src/crawler/classic_crawler.py +++ b/src/crawler/classic_crawler.py @@ -79,7 +79,6 @@ async def parse_feed(user, feed): except Exception as e: up_feed['last_error'] = str(e) up_feed['error_count'] = feed.error_count + 1 - print(up_feed['error_count']) finally: up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal()) if parsed_feed is None: @@ -123,7 +122,7 @@ async def insert_database(user, feed): existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) except Exception as e: - print("existing_article_req: " + str(e)) + logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 @@ -132,15 +131,14 @@ async def insert_database(user, feed): # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) - print("Article %r (%r) already in the database.", - article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: new_updated_date = existing_article.date - print(e) + logger.exception("new_updated_date failed: " + str(e)) + if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace(tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ @@ -157,6 +155,7 @@ async def insert_database(user, feed): art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue + # insertion of the new article article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) |