aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2015-04-08 07:46:38 +0200
committerCédric Bonhomme <cedric@cedricbonhomme.org>2015-04-08 07:46:38 +0200
commit50192b8aab87eebad1df8d699355dae6024c6406 (patch)
treeabd3eadbec56dfdac649fbd4b303adc35fe92484
parentMerged in jaesivsm/pyaggr3g470r (pull request #7) (diff)
downloadnewspipe-50192b8aab87eebad1df8d699355dae6024c6406.tar.gz
newspipe-50192b8aab87eebad1df8d699355dae6024c6406.tar.bz2
newspipe-50192b8aab87eebad1df8d699355dae6024c6406.zip
The minimum error count is now specified in the configuration file.
-rw-r--r--conf.py2
-rw-r--r--conf/conf.cfg-sample1
-rw-r--r--pyaggr3g470r/controllers/feed.py24
-rw-r--r--pyaggr3g470r/crawler.py10
-rw-r--r--pyaggr3g470r/templates/home.html9
-rw-r--r--pyaggr3g470r/views/views.py3
6 files changed, 36 insertions, 13 deletions
diff --git a/conf.py b/conf.py
index 7f3e82ff..0d11cae7 100644
--- a/conf.py
+++ b/conf.py
@@ -48,6 +48,7 @@ if not ON_HEROKU:
HTTP_PROXY = config.get('feedparser', 'http_proxy')
USER_AGENT = config.get('feedparser', 'user_agent')
RESOLVE_ARTICLE_URL = int(config.get('feedparser', 'resolve_article_url')) == 1
+ DEFAULT_MAX_ERROR = int(config.get('feedparser', 'default_max_error'))
WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1
WEBSERVER_HOST = config.get('webserver', 'host')
@@ -77,6 +78,7 @@ else:
HTTP_PROXY = ""
USER_AGENT = "Mozilla/5.0 (X11; Debian; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0"
RESOLVE_ARTICLE_URL = int(os.environ.get('RESOLVE_ARTICLE_URL', 0)) == 1
+ DEFAULT_MAX_ERROR = int(os.environ.get('DEFAULT_MAX_ERROR', 6))
WEBSERVER_DEBUG = False
WEBSERVER_HOST = '0.0.0.0'
diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample
index e5b2683b..b257a236 100644
--- a/conf/conf.cfg-sample
+++ b/conf/conf.cfg-sample
@@ -12,6 +12,7 @@ uri = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/aggregator
http_proxy =
user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r)
resolve_article_url = 0
+default_max_error = 6
[webserver]
debug = 1
host = 0.0.0.0
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index b99a3a7f..8db279ae 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -1,11 +1,33 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
import logging
from datetime import datetime, timedelta
+import conf
from .abstract import AbstractController
from pyaggr3g470r.models import Feed
logger = logging.getLogger(__name__)
-DEFAULT_MAX_ERROR = 6
+DEFAULT_MAX_ERROR = conf.DEFAULT_MAX_ERROR
DEFAULT_LIMIT = 5
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index ded9df6f..c8993c76 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -20,9 +20,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 3.1 $"
+__version__ = "$Revision: 3.2 $"
__date__ = "$Date: 2010/09/02 $"
-__revision__ = "$Date: 2015/02/08 $"
+__revision__ = "$Date: 2015/04/08 $"
__copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "AGPLv3"
@@ -71,9 +71,11 @@ def parse_feed(user, feed):
except Exception as e:
feed.last_error = str(e)
finally:
+ print(feed.link)
if data is None:
+ print('error')
feed.error_count += 1
- if feed.error_count > 2:
+ if feed.error_count >= conf.DEFAULT_MAX_ERROR:
feed.enabled = False
db.session.commit()
return
@@ -83,7 +85,7 @@ def parse_feed(user, feed):
#logger.error(a_feed['bozo_exception'])
feed.last_error = str(a_feed['bozo_exception'])
feed.error_count += 1
- if feed.error_count > 2:
+ if feed.error_count >= conf.DEFAULT_MAX_ERROR:
feed.enabled = False
db.session.commit()
if a_feed['entries'] == []:
diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html
index 69ca582b..c20cacfa 100644
--- a/pyaggr3g470r/templates/home.html
+++ b/pyaggr3g470r/templates/home.html
@@ -1,10 +1,5 @@
{% extends "layout.html" %}
{% block content %}
-<style>
- li.feed-commands {display: none; text-align: right;}
- li.feed-commands > span > a {margin-right: 10px;}
- li.feed-menu:hover + li.feed-commands, li.feed-commands:hover {display: block;}
-</style>
{% if feeds|count == 0 %}
<div class="col-md-4 col-md-offset-4">
<h1>{{ _("You don't have any feeds.") }}</h1>
@@ -22,7 +17,7 @@
<li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
{% if feed_id == fid %}<b>{% endif %}
{% if in_error.get(fid, 0) > 0 %}
- <span style="background-color: {{ "red" if in_error[fid] > 5 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %}</span>
+ <span style="background-color: {{ "red" if in_error[fid] > default_max_error -1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %}</span>
{% endif %}
<span id="unread-{{ fid }}" class="badge pull-right">{{ nbunread }}</span>
{{ feeds[fid]|safe }}
@@ -40,7 +35,7 @@
{% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %}
<li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
{% if in_error.get(fid, 0) > 0 %}
- <span style="background-color: {{ "red" if in_error[fid] > 5 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %}</span>
+ <span style="background-color: {{ "red" if in_error[fid] > default_max_error - 1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %}</span>
{% endif %}
{% if feed_id == fid %}<b>{% endif %}
{{ ftitle|safe }}
diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py
index fd970cba..17d79cab 100644
--- a/pyaggr3g470r/views/views.py
+++ b/pyaggr3g470r/views/views.py
@@ -254,7 +254,8 @@ def home():
return render_template('home.html', gen_url=gen_url, feed_id=feed_id,
filter_=filter_, limit=limit, feeds=feeds,
unread=dict(unread), articles=articles.all(),
- in_error=in_error)
+ in_error=in_error,
+ default_max_error = conf.DEFAULT_MAX_ERROR)
@app.route('/fetch', methods=['GET'])
bgstack15