diff options
| author | François Schmidts <francois.schmidts@gmail.com> | 2016-01-11 10:16:41 +0100 |
|---|---|---|
| committer | François Schmidts <francois.schmidts@gmail.com> | 2016-01-11 10:16:41 +0100 |
| commit | 8a72e6ae786ffde8e841afbe725a19b3b874f87e (patch) | |
| tree | 36d6ff44e909aa00bcb52adbe1f2fe5ede1f53f3 /src/crawler.py | |
| parent | using user agent in web crawler (diff) | |
| download | newspipe-8a72e6ae786ffde8e841afbe725a19b3b874f87e.tar.gz newspipe-8a72e6ae786ffde8e841afbe725a19b3b874f87e.tar.bz2 newspipe-8a72e6ae786ffde8e841afbe725a19b3b874f87e.zip | |
fixing stuffs
* no more warning on constructing feeds
* using the configured user agent for constructing feed
* regrouping the logic behind knowing if the parsing of a feed worked
Diffstat (limited to 'src/crawler.py')
| -rw-r--r-- | src/crawler.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/crawler.py b/src/crawler.py index 22e73754..1a759945 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -37,7 +37,7 @@ import conf from bootstrap import db from web.models import User from web.controllers import FeedController, ArticleController -from web.lib.feed_utils import construct_feed_from +from web.lib.feed_utils import construct_feed_from, is_parsing_ok from web.lib.article_utils import construct_article, extract_id logger = logging.getLogger(__name__) @@ -87,7 +87,7 @@ async def parse_feed(user, feed): FeedController().update({'id': feed.id}, up_feed) return - if parsed_feed['bozo'] == 1 and parsed_feed['entries'] == []: + if not is_parsing_ok(parsed_feed): up_feed['last_error'] = str(parsed_feed['bozo_exception']) up_feed['error_count'] = feed.error_count + 1 FeedController().update({'id': feed.id}, up_feed) |