aboutsummaryrefslogtreecommitdiff
path: root/src/crawler
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2016-02-09 23:05:50 +0100
committerCédric Bonhomme <cedric@cedricbonhomme.org>2016-02-09 23:05:50 +0100
commit47159b1e3818d1a5f1f4eeca15356e99a12c9922 (patch)
tree0505fcdf7ad4924105d5fd0622cc05f05321f1b4 /src/crawler
parentFixed address of the activation link. (diff)
downloadnewspipe-47159b1e3818d1a5f1f4eeca15356e99a12c9922.tar.gz
newspipe-47159b1e3818d1a5f1f4eeca15356e99a12c9922.tar.bz2
newspipe-47159b1e3818d1a5f1f4eeca15356e99a12c9922.zip
The classic crawler is now able to update an article. Only content or title are updatable, as discussed in #27. Need to implement the notification system with the boolean when the content of an article has been updated.
Diffstat (limited to 'src/crawler')
-rw-r--r--src/crawler/classic_crawler.py20
1 files changed, 17 insertions, 3 deletions
diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py
index 1bc37a47..b21a7859 100644
--- a/src/crawler/classic_crawler.py
+++ b/src/crawler/classic_crawler.py
@@ -38,7 +38,8 @@ from bootstrap import db
from web.models import User
from web.controllers import FeedController, ArticleController
from web.lib.feed_utils import construct_feed_from, is_parsing_ok
-from web.lib.article_utils import construct_article, extract_id
+from web.lib.article_utils import construct_article, extract_id, \
+ get_article_content
logger = logging.getLogger(__name__)
@@ -117,11 +118,24 @@ async def insert_database(user, feed):
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
- exist = art_contr.read(feed_id=feed.id,
- **extract_id(article)).count() != 0
+ existing_article_req = art_contr.read(feed_id=feed.id,
+ **extract_id(article))
+ exist = existing_article_req.count() != 0
if exist:
+ existing_article = existing_article_req.first()
+ is_updated = False
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
+ content = get_article_content(article)
+ if existing_article.title != article['title']:
+ existing_article.title = article['title']
+ is_updated = True
+ if existing_article.content != content:
+ existing_article.content = content
+ existing_article.readed = False
+ is_updated = True
+ if is_updated:
+ art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump())
continue
article = construct_article(article, feed)
try:
bgstack15