From dc0cf1a2789293bb3459ea8fe1f611edda075eba Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Sun, 12 Apr 2015 21:44:20 +0200 Subject: redoing conf gathering and using it in templates --- conf.py | 45 +++++++++++++++++++++++++++++--------- conf/conf.cfg-sample | 9 ++++---- pyaggr3g470r/templates/home.html | 4 ++-- pyaggr3g470r/templates/layout.html | 4 ++-- pyaggr3g470r/views/feed.py | 5 ++--- pyaggr3g470r/views/views.py | 7 +++--- 6 files changed, 49 insertions(+), 25 deletions(-) diff --git a/conf.py b/conf.py index 3673850a..5031009c 100644 --- a/conf.py +++ b/conf.py @@ -23,6 +23,26 @@ TIME_ZONE = { } ON_HEROKU = int(os.environ.get('HEROKU', 0)) == 1 +DEFAULTS = {"python": "/usr/bin/python3.4", + "recaptcha_public_key": "", + "recaptcha_private_key": "", + "nb_worker": "100", + "default_max_error": "3", + "log_path": "pyaggr3g470r.log", + "user_agent": "pyAggr3g470r " \ + "(https://bitbucket.org/cedricbonhomme/pyaggr3g470r)", + "resolve_article_url": "false", + "http_proxy": "", + "debug": "true", + "secret": "", + "enabled": "false", + "email": "", + "tls": "false", + "ssl": "true", + "host": "0.0.0.0", + "port": "5000", + "crawling_method": "classic", +} if not ON_HEROKU: try: @@ -30,13 +50,14 @@ if not ON_HEROKU: except: import ConfigParser as confparser # load the configuration - config = confparser.SafeConfigParser() + config = confparser.SafeConfigParser(defaults=DEFAULTS) config.read(os.path.join(basedir, "conf/conf.cfg")) PLATFORM_URL = config.get('misc', 'platform_url') ADMIN_EMAIL = config.get('misc', 'admin_email') RECAPTCHA_PUBLIC_KEY = config.get('misc', 'recaptcha_public_key') - RECAPTCHA_PRIVATE_KEY = config.get('misc', 'recaptcha_private_key') + RECAPTCHA_PRIVATE_KEY = config.get('misc', + 'recaptcha_private_key') LOG_PATH = config.get('misc', 'log_path') PYTHON = config.get('misc', 'python') NB_WORKER = config.getint('misc', 'nb_worker') @@ -47,19 +68,22 @@ if not ON_HEROKU: HTTP_PROXY = config.get('feedparser', 'http_proxy') USER_AGENT = config.get('feedparser', 'user_agent') - RESOLVE_ARTICLE_URL = int(config.get('feedparser', 'resolve_article_url')) == 1 - DEFAULT_MAX_ERROR = int(config.get('feedparser', 'default_max_error')) + RESOLVE_ARTICLE_URL = config.getboolean('feedparser', + 'resolve_article_url') + DEFAULT_MAX_ERROR = config.getint('feedparser', + 'default_max_error') + CRAWLING_METHOD = config.get('feedparser', 'crawling_method') - WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1 + WEBSERVER_DEBUG = config.getboolean('webserver', 'debug') WEBSERVER_HOST = config.get('webserver', 'host') - WEBSERVER_PORT = int(config.get('webserver', 'port')) + WEBSERVER_PORT = config.getint('webserver', 'port') WEBSERVER_SECRET = config.get('webserver', 'secret') NOTIFICATION_EMAIL = config.get('notification', 'email') NOTIFICATION_HOST = config.get('notification', 'host') - NOTIFICATION_PORT = int(config.get('notification', 'port')) - NOTIFICATION_TLS = int(config.get('notification', 'tls')) == 1 - NOTIFICATION_SSL = int(config.get('notification', 'ssl')) == 1 + NOTIFICATION_PORT = config.getint('notification', 'port') + NOTIFICATION_TLS = config.getboolean('notification', 'tls') + NOTIFICATION_SSL = config.getboolean('notification', 'ssl') NOTIFICATION_USERNAME = config.get('notification', 'username') NOTIFICATION_PASSWORD = config.get('notification', 'password') @@ -77,7 +101,8 @@ else: SQLALCHEMY_DATABASE_URI = os.environ['DATABASE_URL'] HTTP_PROXY = "" - USER_AGENT = "Mozilla/5.0 (X11; Debian; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0" + USER_AGENT = "Mozilla/5.0 " \ + "(X11; Debian; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0" RESOLVE_ARTICLE_URL = int(os.environ.get('RESOLVE_ARTICLE_URL', 0)) == 1 DEFAULT_MAX_ERROR = int(os.environ.get('DEFAULT_MAX_ERROR', 6)) diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index b257a236..159af449 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -11,10 +11,11 @@ uri = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/aggregator [feedparser] http_proxy = user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r) -resolve_article_url = 0 +resolve_article_url = false default_max_error = 6 +crawling_method = classic [webserver] -debug = 1 +debug = true host = 0.0.0.0 port = 5000 secret = a secret only you know @@ -22,7 +23,7 @@ secret = a secret only you know email = pyAggr3g470r@no-reply.com host = smtp.googlemail.com port = 465 -tls = 0 -ssl = 1 +tls = false +ssl = true username = your-gmail-username password = your-gmail-password diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html index 98b12e25..461b6928 100644 --- a/pyaggr3g470r/templates/home.html +++ b/pyaggr3g470r/templates/home.html @@ -17,7 +17,7 @@
  • {% if feed_id == fid %}{% endif %} {% if in_error.get(fid, 0) > 0 %} - default_max_error -1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} + conf.DEFAULT_MAX_ERROR -1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} {% endif %} {{ nbunread }} {{ feeds[fid]|safe }} @@ -35,7 +35,7 @@ {% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %}
  • {% if in_error.get(fid, 0) > 0 %} - default_max_error - 1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} + conf.DEFAULT_MAX_ERROR - 1 else "orange" }} ;" class="badge pull-right" title="Some errors occured while trying to retrieve that feed.">{{ in_error[fid] }} {{ _("error") }}{% if in_error[fid] > 1 %}s{% endif %} {% endif %} {% if feed_id == fid %}{% endif %} {{ ftitle|safe }} diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html index e3d3be38..484bbdc7 100644 --- a/pyaggr3g470r/templates/layout.html +++ b/pyaggr3g470r/templates/layout.html @@ -65,7 +65,7 @@ {% else %}
  • {{ _('Favorites') }}
    • {% endif %} - {% if not on_heroku or g.user.is_admin() %} + {% if conf.CRAWLING_METHOD == "classic" and (not conf.ON_HEROKU or g.user.is_admin()) %}
  • {{ _('Fetch') }}
    • {% endif %}
  • @@ -94,7 +94,7 @@
    • - {% if not on_heroku %} + {% if not conf.ON_HEROKU %}
  •  
    diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py index 846df06d..11d14ac2 100644 --- a/pyaggr3g470r/views/feed.py +++ b/pyaggr3g470r/views/feed.py @@ -175,9 +175,8 @@ def form(feed_id=None): form = AddFeedForm(obj=feed) return render_template('edit_feed.html', action=gettext("Edit the feed"), - form=form, feed=feed, - not_on_heroku=not conf.ON_HEROKU) + form=form, feed=feed) # Return an empty form in order to create a new feed return render_template('edit_feed.html', action=gettext("Add a feed"), - form=form, not_on_heroku=not conf.ON_HEROKU) + form=form) diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 1918b1b5..e70a8c6f 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -139,6 +139,7 @@ def get_timezone(): except: return conf.TIME_ZONE["en"] + @app.context_processor def inject_feed_form(): """ @@ -147,8 +148,7 @@ def inject_feed_form(): Context processors run before the template is rendered and have the ability to inject new values into the template context. """ - return dict(create_feed_form=AddFeedForm(), - on_heroku=conf.ON_HEROKU) + return dict(create_feed_form=AddFeedForm()) # # Views. @@ -285,8 +285,7 @@ def home(favorites=False): return render_template('home.html', gen_url=gen_url, feed_id=feed_id, filter_=filter_, limit=limit, feeds=feeds, unread=unread, articles=articles, in_error=in_error, - head_title=head_title, favorites=favorites, - default_max_error = conf.DEFAULT_MAX_ERROR) + head_title=head_title, favorites=favorites) @app.route('/favorites') -- cgit From b8416422b53a8b99738fef712a1857cdbbf8b887 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 14 Apr 2015 08:53:40 +0200 Subject: making fetch call dependent to the crawling method option --- pyaggr3g470r/views/feed.py | 3 ++- pyaggr3g470r/views/views.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py index 11d14ac2..159dce64 100644 --- a/pyaggr3g470r/views/feed.py +++ b/pyaggr3g470r/views/feed.py @@ -163,7 +163,8 @@ def form(feed_id=None): flash(gettext('Feed %(feed_title)r successfully created.', feed_title=new_feed.title), 'success') - utils.fetch(g.user.id, new_feed.id) + if conf.CRAWLING_METHOD == "classic": + utils.fetch(g.user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), 'info') return redirect(url_for('feed.form', diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index e70a8c6f..d0c06da0 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -302,7 +302,8 @@ def fetch(feed_id=None): Triggers the download of news. News are downloaded in a separated process, mandatory for Heroku. """ - if not conf.ON_HEROKU or g.user.is_admin(): + if conf.CRAWLING_METHOD == "classic" \ + and (not conf.ON_HEROKU or g.user.is_admin()): utils.fetch(g.user.id, feed_id) flash(gettext("Downloading articles..."), "info") else: @@ -519,11 +520,14 @@ def management(): else: try: nb = utils.import_opml(g.user.email, data.read()) - utils.fetch(g.user.email, None) - flash(str(nb) + ' ' + gettext('feeds imported.'), "success") - flash(gettext("Downloading articles..."), 'info') + if conf.CRAWLING_METHOD == "classic": + utils.fetch(g.user.email, None) + flash(str(nb) + ' ' + gettext('feeds imported.'), + "success") + flash(gettext("Downloading articles..."), 'info') except: - flash(gettext("Impossible to import the new feeds."), "danger") + flash(gettext("Impossible to import the new feeds."), + "danger") elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) @@ -534,7 +538,8 @@ def management(): nb = utils.import_json(g.user.email, data.read()) flash(gettext('Account imported.'), "success") except: - flash(gettext("Impossible to import the account."), "danger") + flash(gettext("Impossible to import the account."), + "danger") else: flash(gettext('File not allowed.'), 'danger') -- cgit