aboutsummaryrefslogtreecommitdiff
path: root/src/crawler
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2016-10-28 08:07:39 +0200
committerCédric Bonhomme <cedric@cedricbonhomme.org>2016-10-28 08:07:39 +0200
commitc9f812335a6bf89d19a78940b02c39b22c0af4b5 (patch)
tree45baf34cb37fcc04f1e205082b6ae74d37ecf24a /src/crawler
parenttest (diff)
downloadnewspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.tar.gz
newspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.tar.bz2
newspipe-c9f812335a6bf89d19a78940b02c39b22c0af4b5.zip
a mess
Diffstat (limited to 'src/crawler')
-rw-r--r--src/crawler/classic_crawler.py9
1 files changed, 4 insertions, 5 deletions
diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py
index fcbbe22b..4125dad7 100644
--- a/src/crawler/classic_crawler.py
+++ b/src/crawler/classic_crawler.py
@@ -79,7 +79,6 @@ async def parse_feed(user, feed):
except Exception as e:
up_feed['last_error'] = str(e)
up_feed['error_count'] = feed.error_count + 1
- print(up_feed['error_count'])
finally:
up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
@@ -123,7 +122,7 @@ async def insert_database(user, feed):
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
except Exception as e:
- print("existing_article_req: " + str(e))
+ logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
@@ -132,15 +131,14 @@ async def insert_database(user, feed):
# the content or the title
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
- print("Article %r (%r) already in the database.",
- article['title'], article['link'])
existing_article = existing_article_req.first()
new_updated_date = None
try:
new_updated_date = dateutil.parser.parse(article['updated'])
except Exception as e:
new_updated_date = existing_article.date
- print(e)
+ logger.exception("new_updated_date failed: " + str(e))
+
if None is existing_article.updated_date:
existing_article.updated_date = new_updated_date.replace(tzinfo=None)
if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
@@ -157,6 +155,7 @@ async def insert_database(user, feed):
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
+ # insertion of the new article
article = construct_article(article, feed)
try:
new_articles.append(art_contr.create(**article))
bgstack15