From 50192b8aab87eebad1df8d699355dae6024c6406 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Wed, 8 Apr 2015 07:46:38 +0200 Subject: The minimum error count is now specified in the configuration file. --- conf.py | 2 ++ conf/conf.cfg-sample | 1 + pyaggr3g470r/controllers/feed.py | 24 +++++++++++++++++++++++- pyaggr3g470r/crawler.py | 10 ++++++---- pyaggr3g470r/templates/home.html | 9 ++------- pyaggr3g470r/views/views.py | 3 ++- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/conf.py b/conf.py index 7f3e82ff..0d11cae7 100644 --- a/conf.py +++ b/conf.py @@ -48,6 +48,7 @@ if not ON_HEROKU: HTTP_PROXY = config.get('feedparser', 'http_proxy') USER_AGENT = config.get('feedparser', 'user_agent') RESOLVE_ARTICLE_URL = int(config.get('feedparser', 'resolve_article_url')) == 1 + DEFAULT_MAX_ERROR = int(config.get('feedparser', 'default_max_error')) WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1 WEBSERVER_HOST = config.get('webserver', 'host') @@ -77,6 +78,7 @@ else: HTTP_PROXY = "" USER_AGENT = "Mozilla/5.0 (X11; Debian; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0" RESOLVE_ARTICLE_URL = int(os.environ.get('RESOLVE_ARTICLE_URL', 0)) == 1 + DEFAULT_MAX_ERROR = int(os.environ.get('DEFAULT_MAX_ERROR', 6)) WEBSERVER_DEBUG = False WEBSERVER_HOST = '0.0.0.0' diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index e5b2683b..b257a236 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -12,6 +12,7 @@ uri = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/aggregator http_proxy = user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r) resolve_article_url = 0 +default_max_error = 6 [webserver] debug = 1 host = 0.0.0.0 diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index b99a3a7f..8db279ae 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -1,11 +1,33 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import logging from datetime import datetime, timedelta +import conf from .abstract import AbstractController from pyaggr3g470r.models import Feed logger = logging.getLogger(__name__) -DEFAULT_MAX_ERROR = 6 +DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR DEFAULT_LIMIT = 5 diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index ded9df6f..c8993c76 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -20,9 +20,9 @@ # along with this program. If not, see . __author__ = "Cedric Bonhomme" -__version__ = "$Revision: 3.1 $" +__version__ = "$Revision: 3.2 $" __date__ = "$Date: 2010/09/02 $" -__revision__ = "$Date: 2015/02/08 $" +__revision__ = "$Date: 2015/04/08 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "AGPLv3" @@ -71,9 +71,11 @@ def parse_feed(user, feed): except Exception as e: feed.last_error = str(e) finally: + print(feed.link) if data is None: + print('error') feed.error_count += 1 - if feed.error_count > 2: + if feed.error_count >= conf.DEFAULT_MAX_ERROR: feed.enabled = False db.session.commit() return @@ -83,7 +85,7 @@ def parse_feed(user, feed): #logger.error(a_feed['bozo_exception']) feed.last_error = str(a_feed['bozo_exception']) feed.error_count += 1 - if feed.error_count > 2: + if feed.error_count >= conf.DEFAULT_MAX_ERROR: feed.enabled = False db.session.commit() if a_feed['entries'] == []: diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html index 69ca582b..c20cacfa 100644 --- a/pyaggr3g470r/templates/home.html +++ b/pyaggr3g470r/templates/home.html @@ -1,10 +1,5 @@ {% extends "layout.html" %} {% block content %} - {% if feeds|count == 0 %}

{{ _("You don't have any feeds.") }}

@@ -22,7 +17,7 @@
  • {% if feed_id == fid %}{% endif %} {% if in_error.get(fid, 0) > 0 %} - 5 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} + default_max_error -1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} {% endif %} {{ nbunread }} {{ feeds[fid]|safe }} @@ -40,7 +35,7 @@ {% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %}
  • {% if in_error.get(fid, 0) > 0 %} - 5 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} + default_max_error - 1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} {% endif %} {% if feed_id == fid %}{% endif %} {{ ftitle|safe }} diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index fd970cba..17d79cab 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -254,7 +254,8 @@ def home(): return render_template('home.html', gen_url=gen_url, feed_id=feed_id, filter_=filter_, limit=limit, feeds=feeds, unread=dict(unread), articles=articles.all(), - in_error=in_error) + in_error=in_error, + default_max_error = conf.DEFAULT_MAX_ERROR) @app.route('/fetch', methods=['GET']) -- cgit