diff options
| author | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2016-01-11 15:22:26 +0100 |
|---|---|---|
| committer | Cédric Bonhomme <cedric@cedricbonhomme.org> | 2016-01-11 15:22:26 +0100 |
| commit | 6d0e7c8fe0da4ce96fb9c7df3ab9c970f3a2913d (patch) | |
| tree | 36d6ff44e909aa00bcb52adbe1f2fe5ede1f53f3 /src/crawler.py | |
| parent | Moved the crawler in the parent folder. (diff) | |
| parent | fixing stuffs (diff) | |
| download | newspipe-6d0e7c8fe0da4ce96fb9c7df3ab9c970f3a2913d.tar.gz newspipe-6d0e7c8fe0da4ce96fb9c7df3ab9c970f3a2913d.tar.bz2 newspipe-6d0e7c8fe0da4ce96fb9c7df3ab9c970f3a2913d.zip | |
Merge pull request #30 from jaesivsm/master
misc fixes on crawlers and use of User-Agent
Diffstat (limited to 'src/crawler.py')
| -rw-r--r-- | src/crawler.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/src/crawler.py b/src/crawler.py index 22e73754..1a759945 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -37,7 +37,7 @@ import conf from bootstrap import db from web.models import User from web.controllers import FeedController, ArticleController -from web.lib.feed_utils import construct_feed_from +from web.lib.feed_utils import construct_feed_from, is_parsing_ok from web.lib.article_utils import construct_article, extract_id logger = logging.getLogger(__name__) @@ -87,7 +87,7 @@ async def parse_feed(user, feed): FeedController().update({'id': feed.id}, up_feed) return - if parsed_feed['bozo'] == 1 and parsed_feed['entries'] == []: + if not is_parsing_ok(parsed_feed): up_feed['last_error'] = str(parsed_feed['bozo_exception']) up_feed['error_count'] = feed.error_count + 1 FeedController().update({'id': feed.id}, up_feed) |