diff options
| -rw-r--r-- | src/crawler/default_crawler.py | 20 |
1 files changed, 0 insertions, 20 deletions
diff --git a/src/crawler/default_crawler.py b/src/crawler/default_crawler.py index bba8431a..e511eb9d 100644 --- a/src/crawler/default_crawler.py +++ b/src/crawler/default_crawler.py @@ -141,26 +141,6 @@ async def insert_articles(queue, nḅ_producers=1): exist = existing_article_req.count() != 0 if exist: continue - # if the article has been already retrieved, we only update - # the content or the title - logger.info('Article already in the database: {}'. \ - format(article['link'])) - existing_article = existing_article_req.first() - - if new_article['date'].replace(tzinfo=None) != \ - existing_article.date: - existing_article.date = new_article['date'] - existing_article.updated_date = new_article['date'] - if existing_article.title != new_article['title']: - existing_article.title = new_article['title'] - content = get_article_content(article) - if existing_article.content != content: - existing_article.content = content - existing_article.readed = False - art_contr.update({'entry_id': existing_article.entry_id}, - existing_article.dump()) - logger.info('Article updated: {}'.format(article['link'])) - continue # insertion of the new article try: |