aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-08-03 23:55:46 +0200
committerCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-08-03 23:55:46 +0200
commitcdbd573500a365e290e88b50d7b0c2355b7f7e19 (patch)
tree25ede52ae4b02a2377ae40d2c146c7ed2e9abe2a
parentThe numver ov values of the splited string is variable (sometimes the charset... (diff)
parentsqlalchemy was requesting icons everytime feed where listed (diff)
downloadnewspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.tar.gz
newspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.tar.bz2
newspipe-cdbd573500a365e290e88b50d7b0c2355b7f7e19.zip
Merged in jaesivsm/pyaggr3g470r (pull request #20)
perf improvement
-rw-r--r--conf.py4
-rw-r--r--conf/conf.cfg-sample2
-rwxr-xr-xmanager.py10
-rw-r--r--migrations/versions/19bdaa6208e_add_icon_column.py2
-rw-r--r--migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py36
-rw-r--r--pyaggr3g470r/controllers/__init__.py4
-rw-r--r--pyaggr3g470r/controllers/abstract.py10
-rw-r--r--pyaggr3g470r/controllers/article.py6
-rw-r--r--pyaggr3g470r/controllers/feed.py19
-rw-r--r--pyaggr3g470r/controllers/icon.py23
-rw-r--r--pyaggr3g470r/lib/crawler.py9
-rw-r--r--pyaggr3g470r/lib/feed_utils.py33
-rw-r--r--pyaggr3g470r/lib/utils.py11
-rw-r--r--pyaggr3g470r/lib/view_utils.py26
-rw-r--r--pyaggr3g470r/models/__init__.py3
-rw-r--r--pyaggr3g470r/models/feed.py8
-rw-r--r--pyaggr3g470r/models/icon.py7
-rw-r--r--pyaggr3g470r/templates/feeds.html2
-rw-r--r--pyaggr3g470r/templates/home.html4
-rw-r--r--pyaggr3g470r/views/__init__.py1
-rw-r--r--pyaggr3g470r/views/api/feed.py4
-rw-r--r--pyaggr3g470r/views/article.py2
-rw-r--r--pyaggr3g470r/views/feed.py21
-rw-r--r--pyaggr3g470r/views/icon.py14
-rw-r--r--pyaggr3g470r/views/views.py26
-rwxr-xr-xrunserver.py1
-rw-r--r--scripts/__init__.py0
-rw-r--r--scripts/probes.py74
28 files changed, 286 insertions, 76 deletions
diff --git a/conf.py b/conf.py
index 5704ab83..73e83eca 100644
--- a/conf.py
+++ b/conf.py
@@ -28,6 +28,8 @@ DEFAULTS = {"platform_url": "https://pyaggr3g470r.herokuapp.com/",
"recaptcha_public_key": "",
"recaptcha_private_key": "",
"nb_worker": "100",
+ "api_login": "",
+ "api_passwd": "",
"default_max_error": "3",
"log_path": "pyaggr3g470r.log",
"log_level": "info",
@@ -79,6 +81,8 @@ RECAPTCHA_PRIVATE_KEY = config.get('misc',
'recaptcha_private_key')
LOG_PATH = config.get('misc', 'log_path')
NB_WORKER = config.getint('misc', 'nb_worker')
+API_LOGIN = config.get('crawler', 'api_login')
+API_PASSWD = config.get('crawler', 'api_passwd')
WHOOSH_ENABLED = True
diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample
index 2f30b04a..acd244c8 100644
--- a/conf/conf.cfg-sample
+++ b/conf/conf.cfg-sample
@@ -18,6 +18,8 @@ default_max_error = 6
http_proxy =
user_agent = pyAggr3g470r (https://bitbucket.org/cedricbonhomme/pyaggr3g470r)
resolve_article_url = false
+api_login =
+api_passwd =
[notification]
notification_email = pyAggr3g470r@no-reply.com
host = smtp.googlemail.com
diff --git a/manager.py b/manager.py
index 1c038dd6..272dfc2e 100755
--- a/manager.py
+++ b/manager.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
-from bootstrap import application, db, populate_g
+from bootstrap import application, db, populate_g, conf
from flask.ext.script import Manager
from flask.ext.migrate import Migrate, MigrateCommand
@@ -27,10 +27,10 @@ def db_create():
pyaggr3g470r.models.db_create(db)
@manager.command
-def fetch(user, password, limit=100, retreive_all=False):
+def fetch(limit=100, retreive_all=False):
"Crawl the feeds with the client crawler."
from pyaggr3g470r.lib.crawler import CrawlerScheduler
- scheduler = CrawlerScheduler(user, password)
+ scheduler = CrawlerScheduler(conf.API_LOGIN, conf.API_PASSWD)
scheduler.run(limit=limit, retreive_all=retreive_all)
scheduler.wait()
@@ -66,5 +66,9 @@ def fetch_asyncio(user_id, feed_id):
feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
loop.close()
+from scripts.probes import ArticleProbe, FeedProbe
+manager.add_command('probe_articles', ArticleProbe())
+manager.add_command('probe_feeds', FeedProbe())
+
if __name__ == '__main__':
manager.run()
diff --git a/migrations/versions/19bdaa6208e_add_icon_column.py b/migrations/versions/19bdaa6208e_add_icon_column.py
index 5762d6d7..2efa376f 100644
--- a/migrations/versions/19bdaa6208e_add_icon_column.py
+++ b/migrations/versions/19bdaa6208e_add_icon_column.py
@@ -1,4 +1,4 @@
-"""empty message
+"""adding icon columns
Revision ID: 19bdaa6208e
Revises: 422da2d0234
diff --git a/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py b/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py
new file mode 100644
index 00000000..2c8eeda5
--- /dev/null
+++ b/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py
@@ -0,0 +1,36 @@
+"""moving icons to their own table
+
+Revision ID: 25ca960a207
+Revises: 19bdaa6208e
+Create Date: 2015-08-03 14:36:21.626411
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '25ca960a207'
+down_revision = '19bdaa6208e'
+
+from alembic import op
+import sqlalchemy as sa
+
+import conf
+
+
+def upgrade():
+ op.create_table('icon',
+ sa.Column('url', sa.String(), nullable=False),
+ sa.Column('content', sa.String(), nullable=True),
+ sa.Column('mimetype', sa.String(), nullable=True),
+ sa.PrimaryKeyConstraint('url'))
+ op.add_column('feed', sa.Column('icon_url', sa.String(), nullable=True))
+ if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI:
+ op.create_foreign_key(None, 'feed', 'icon', ['icon_url'], ['url'])
+ op.drop_column('feed', 'icon')
+
+
+def downgrade():
+ op.add_column('feed', sa.Column('icon', sa.VARCHAR(), nullable=True))
+ if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI:
+ op.drop_constraint(None, 'feed', type_='foreignkey')
+ op.drop_column('feed', 'icon_url')
+ op.drop_table('icon')
diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py
index d8d1a104..ad77fa1d 100644
--- a/pyaggr3g470r/controllers/__init__.py
+++ b/pyaggr3g470r/controllers/__init__.py
@@ -1,6 +1,8 @@
from .feed import FeedController
from .article import ArticleController
from .user import UserController
+from .icon import IconController
-__all__ = ['FeedController', 'ArticleController', 'UserController']
+__all__ = ['FeedController', 'ArticleController', 'UserController',
+ 'IconController']
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
index 281e1415..f33d241e 100644
--- a/pyaggr3g470r/controllers/abstract.py
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -65,7 +65,8 @@ class AbstractController(object):
dependant) and the user is not an admin and the filters doesn't already
contains a filter for that user.
"""
- if self.user_id and filters.get(self._user_id_key) != self.user_id:
+ if self._user_id_key is not None and self.user_id \
+ and filters.get(self._user_id_key) != self.user_id:
filters[self._user_id_key] = self.user_id
return self._db_cls.query.filter(*self._to_filters(**filters))
@@ -82,10 +83,11 @@ class AbstractController(object):
return obj
def create(self, **attrs):
- assert self._user_id_key in attrs or self.user_id is not None, \
+ assert self._user_id_key is None or self._user_id_key in attrs \
+ or self.user_id is not None, \
"You must provide user_id one way or another"
- if self._user_id_key not in attrs:
+ if self._user_id_key is not None and self._user_id_key not in attrs:
attrs[self._user_id_key] = self.user_id
obj = self._db_cls(**attrs)
db.session.add(obj)
@@ -108,5 +110,7 @@ class AbstractController(object):
def _has_right_on(self, obj):
# user_id == None is like being admin
+ if self._user_id_key is None:
+ return True
return self.user_id is None \
or getattr(obj, self._user_id_key, None) == self.user_id
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
index 70b9d2dd..21b4b5e7 100644
--- a/pyaggr3g470r/controllers/article.py
+++ b/pyaggr3g470r/controllers/article.py
@@ -33,6 +33,12 @@ class ArticleController(AbstractController):
.filter(*self._to_filters(**filters))
.group_by(Article.feed_id).all())
+ def count_by_user_id(self, **filters):
+ return dict(db.session.query(Article.user_id,
+ func.count(Article.id))
+ .filter(*self._to_filters(**filters))
+ .group_by(Article.user_id).all())
+
def create(self, **attrs):
# handling special denorm for article rights
assert 'feed_id' in attrs
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
index 82714e39..6b3c4fb5 100644
--- a/pyaggr3g470r/controllers/feed.py
+++ b/pyaggr3g470r/controllers/feed.py
@@ -21,9 +21,11 @@
import logging
from datetime import datetime, timedelta
+from werkzeug.exceptions import NotFound
import conf
from .abstract import AbstractController
+from .icon import IconController
from pyaggr3g470r.models import Feed
logger = logging.getLogger(__name__)
@@ -52,3 +54,20 @@ class FeedController(AbstractController):
self.update({'id__in': [feed.id for feed in feeds]},
{'last_retrieved': now})
return feeds
+
+ def _ensure_icon(self, attrs):
+ if not attrs.get('icon_url'):
+ return
+ icon_contr = IconController()
+ try:
+ icon_contr.get(url=attrs['icon_url'])
+ except NotFound:
+ icon_contr.create(**{'url': attrs['icon_url']})
+
+ def create(self, **attrs):
+ self._ensure_icon(attrs)
+ return super().create(**attrs)
+
+ def update(self, filters, attrs):
+ self._ensure_icon(attrs)
+ return super().update(filters, attrs)
diff --git a/pyaggr3g470r/controllers/icon.py b/pyaggr3g470r/controllers/icon.py
new file mode 100644
index 00000000..194c601c
--- /dev/null
+++ b/pyaggr3g470r/controllers/icon.py
@@ -0,0 +1,23 @@
+import base64
+import requests
+from pyaggr3g470r.models import Icon
+from .abstract import AbstractController
+
+
+class IconController(AbstractController):
+ _db_cls = Icon
+ _user_id_key = None
+
+ def _build_from_url(self, attrs):
+ if 'url' in attrs and 'content' not in attrs:
+ resp = requests.get(attrs['url'], verify=False)
+ attrs.update({'url': resp.url,
+ 'mimetype': resp.headers.get('content-type', None),
+ 'content': base64.b64encode(resp.content).decode('utf8')})
+ return attrs
+
+ def create(self, **attrs):
+ return super().create(**self._build_from_url(attrs))
+
+ def update(self, filters, attrs):
+ return super().update(filters, self._build_from_url(attrs))
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index e5998776..216e7a96 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -125,7 +125,7 @@ class PyAggUpdater(AbstractCrawler):
entry = construct_article(
self.entries[tuple(sorted(id_to_create.items()))],
self.feed)
- logger.warn('%r %r - creating %r for %r - %r', self.feed['id'],
+ logger.info('%r %r - creating %r for %r - %r', self.feed['id'],
self.feed['title'], entry['title'], entry['user_id'],
id_to_create)
self.query_pyagg('post', 'article', entry)
@@ -141,7 +141,7 @@ class PyAggUpdater(AbstractCrawler):
strftime('%a, %d %b %Y %X %Z', gmtime()))}
fresh_feed = construct_feed_from(url=self.feed['link'],
fp_parsed=self.parsed_feed)
- for key in ('description', 'site_link', 'icon'):
+ for key in ('description', 'site_link', 'icon_url'):
if fresh_feed.get(key) and fresh_feed[key] != self.feed.get(key):
up_feed[key] = fresh_feed[key]
if not self.feed.get('title'):
@@ -152,11 +152,12 @@ class PyAggUpdater(AbstractCrawler):
up_feed['last_retrieved'] \
= (datetime.now() - timedelta(minutes=45)).isoformat()
- logger.info('%r %r - pushing feed attrs %r',
+ if any([up_feed[key] != self.feed.get(key) for key in up_feed]):
+ logger.warn('%r %r - pushing feed attrs %r',
self.feed['id'], self.feed['title'],
{key: "%s -> %s" % (up_feed[key], self.feed.get(key))
for key in up_feed if up_feed[key] != self.feed.get(key)})
- if any([up_feed[key] != self.feed.get(key) for key in up_feed]):
+
future = self.query_pyagg('put',
'feed/%d' % self.feed['id'], up_feed)
future.add_done_callback(self.get_counter_callback())
diff --git a/pyaggr3g470r/lib/feed_utils.py b/pyaggr3g470r/lib/feed_utils.py
index 28123f66..aa9db29c 100644
--- a/pyaggr3g470r/lib/feed_utils.py
+++ b/pyaggr3g470r/lib/feed_utils.py
@@ -4,7 +4,7 @@ import requests
import feedparser
from bs4 import BeautifulSoup, SoupStrainer
-from pyaggr3g470r.lib.utils import try_keys, try_get_b64icon, rebuild_url
+from pyaggr3g470r.lib.utils import try_keys, try_get_icon_url, rebuild_url
logger = logging.getLogger(__name__)
@@ -29,7 +29,7 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link')
feed['title'] = fp_parsed['feed'].get('title')
feed['description'] = try_keys(fp_parsed['feed'], 'subtitle', 'title')
- feed['icon'] = try_keys(fp_parsed['feed'], 'icon')
+ feed['icon_url'] = try_keys(fp_parsed['feed'], 'icon')
else:
feed['site_link'] = url
@@ -37,13 +37,14 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
feed['site_link'] = rebuild_url(feed['site_link'], feed_split)
site_split = urllib.parse.urlsplit(feed['site_link'])
- if feed.get('icon'):
- feed['icon'] = try_get_b64icon(feed['icon'], site_split, feed_split)
- if feed['icon'] is None:
- del feed['icon']
+ if feed.get('icon_url'):
+ feed['icon_url'] = try_get_icon_url(
+ feed['icon_url'], site_split, feed_split)
+ if feed['icon_url'] is None:
+ del feed['icon_url']
if not feed.get('site_link') or not query_site \
- or all(bool(feed.get(key)) for key in ('link', 'title', 'icon')):
+ or all(bool(feed.get(k)) for k in ('link', 'title', 'icon_url')):
return feed
response = requests.get(feed['site_link'], verify=False)
@@ -66,22 +67,22 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
return True
return wrapper
- if not feed.get('icon'):
+ if not feed.get('icon_url'):
icons = bs_parsed.find_all(check_keys(rel=['icon', 'shortcut']))
if not len(icons):
icons = bs_parsed.find_all(check_keys(rel=['icon']))
if len(icons) >= 1:
for icon in icons:
- feed['icon'] = try_get_b64icon(icon.attrs['href'],
- site_split, feed_split)
- if feed['icon'] is not None:
+ feed['icon_url'] = try_get_icon_url(icon.attrs['href'],
+ site_split, feed_split)
+ if feed['icon_url'] is not None:
break
- if feed.get('icon') is None:
- feed['icon'] = try_get_b64icon('/favicon.ico',
- site_split, feed_split)
- if 'icon' in feed and feed['icon'] is None:
- del feed['icon']
+ if feed.get('icon_url') is None:
+ feed['icon_url'] = try_get_icon_url('/favicon.ico',
+ site_split, feed_split)
+ if 'icon_url' in feed and feed['icon_url'] is None:
+ del feed['icon_url']
if not feed.get('link'):
alternates = bs_parsed.find_all(check_keys(rel=['alternate'],
diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py
index a51b6c3e..aa552a12 100644
--- a/pyaggr3g470r/lib/utils.py
+++ b/pyaggr3g470r/lib/utils.py
@@ -1,6 +1,5 @@
import types
import urllib
-import base64
import logging
import requests
from hashlib import md5
@@ -40,7 +39,7 @@ def rebuild_url(url, base_split):
return urllib.parse.urlunsplit(new_split)
-def try_get_b64icon(url, *splits):
+def try_get_icon_url(url, *splits):
for split in splits:
if split is None:
continue
@@ -48,11 +47,11 @@ def try_get_b64icon(url, *splits):
response = requests.get(rb_url, verify=False, timeout=10)
# if html in content-type, we assume it's a fancy 404 page
content_type = response.headers.get('content-type', '')
- if response.ok and 'html' not in content_type:
- return content_type + (
- '\n%s' % base64.b64encode(response.content).decode('utf8'))
+ if response.ok and 'html' not in content_type and response.content:
+ return response.url
return None
def to_hash(text):
- return md5(text.encode('utf8')).hexdigest()
+ return md5(text.encode('utf8') if hasattr(text, 'encode') else text)\
+ .hexdigest()
diff --git a/pyaggr3g470r/lib/view_utils.py b/pyaggr3g470r/lib/view_utils.py
new file mode 100644
index 00000000..0cfe62c4
--- /dev/null
+++ b/pyaggr3g470r/lib/view_utils.py
@@ -0,0 +1,26 @@
+from functools import wraps
+from flask import request, Response, make_response
+from pyaggr3g470r.lib.utils import to_hash
+
+
+def etag_match(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ response = func(*args, **kwargs)
+ if isinstance(response, Response):
+ etag = to_hash(response.data)
+ headers = response.headers
+ elif type(response) is str:
+ etag = to_hash(response)
+ headers = {}
+ else:
+ return response
+ if request.headers.get('if-none-match') == etag:
+ response = Response(status=304)
+ response.headers['Cache-Control'] \
+ = headers.get('Cache-Control', 'pragma: no-cache')
+ elif not isinstance(response, Response):
+ response = make_response(response)
+ response.headers['etag'] = etag
+ return response
+ return wrapper
diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py
index ba52b0de..b578094c 100644
--- a/pyaggr3g470r/models/__init__.py
+++ b/pyaggr3g470r/models/__init__.py
@@ -30,8 +30,9 @@ from .feed import Feed
from .role import Role
from .user import User
from .article import Article
+from .icon import Icon
-__all__ = ['Feed', 'Role', 'User', 'Article']
+__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon']
import os
diff --git a/pyaggr3g470r/models/feed.py b/pyaggr3g470r/models/feed.py
index 75e55df1..07b0fc99 100644
--- a/pyaggr3g470r/models/feed.py
+++ b/pyaggr3g470r/models/feed.py
@@ -35,7 +35,7 @@ class Feed(db.Model):
"""
Represent a feed.
"""
- id = db.Column(db.Integer, primary_key=True)
+ id = db.Column(db.Integer(), primary_key=True)
title = db.Column(db.String(), default="")
description = db.Column(db.String(), default="FR")
link = db.Column(db.String())
@@ -43,7 +43,6 @@ class Feed(db.Model):
enabled = db.Column(db.Boolean(), default=True)
created_date = db.Column(db.DateTime(), default=datetime.now)
filters = db.Column(db.PickleType, default=[])
- icon = db.Column(db.String(), default="")
# cache handling
etag = db.Column(db.String(), default="")
@@ -55,7 +54,8 @@ class Feed(db.Model):
error_count = db.Column(db.Integer(), default=0)
# relationship
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+ icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
+ user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
articles = db.relationship('Article', backref='source', lazy='dynamic',
cascade='all,delete-orphan',
order_by=desc("Article.date"))
@@ -71,7 +71,7 @@ class Feed(db.Model):
"link": self.link,
"site_link": self.site_link,
"etag": self.etag,
- "icon": self.icon,
+ "icon_url": self.icon_url,
"error_count": self.error_count,
"last_modified": self.last_modified,
"last_retrieved": self.last_retrieved}
diff --git a/pyaggr3g470r/models/icon.py b/pyaggr3g470r/models/icon.py
new file mode 100644
index 00000000..22ef1164
--- /dev/null
+++ b/pyaggr3g470r/models/icon.py
@@ -0,0 +1,7 @@
+from bootstrap import db
+
+
+class Icon(db.Model):
+ url = db.Column(db.String(), primary_key=True)
+ content = db.Column(db.String(), default=None)
+ mimetype = db.Column(db.String(), default="application/image")
diff --git a/pyaggr3g470r/templates/feeds.html b/pyaggr3g470r/templates/feeds.html
index 20e0cccb..82af2411 100644
--- a/pyaggr3g470r/templates/feeds.html
+++ b/pyaggr3g470r/templates/feeds.html
@@ -29,7 +29,7 @@
{% endif %}
</td>
<td><a href="{{ url_for("feed.feed", feed_id=feed.id) }}" {% if feed.description %}title="{{ feed.description }}"{% endif %}>
- {% if feed.icon %}<img src="{{ url_for('feed.icon', feed_id=feed.id) }}" width="16px" />{% endif %}
+ {% if feed.icon_url %}<img src="{{ url_for('icon.icon', url=feed.icon_url) }}" width="16px" />{% endif %}
{{ feed.title }}
</a></td>
<td><a href="{{ feed.site_link }}">{{ feed.site_link }}</a></td>
diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html
index 1577bd32..86d96e94 100644
--- a/pyaggr3g470r/templates/home.html
+++ b/pyaggr3g470r/templates/home.html
@@ -107,8 +107,8 @@
</td>
<td>
<a class="open-article" href="{{ url_for("article.redirect_to_article", article_id=article.id)}}" target="_blank" title="{{article.link}}" alt="{{article.link}}">
- {% if article.source.icon %}
- <img src="{{ url_for('feed.icon', feed_id=article.feed_id) }}" width="16px" />
+ {% if article.source.icon_url %}
+ <img src="{{ url_for('icon.icon', url=article.source.icon_url) }}" width="16px" />
{% else %}
<span class="glyphicon glyphicon-ban-circle" title='{{_("No icon found for this feed")}}' alt='{{_("No icon found for this feed")}}'></span>
{% endif %}
diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py
index 029dcb7d..36d382bd 100644
--- a/pyaggr3g470r/views/__init__.py
+++ b/pyaggr3g470r/views/__init__.py
@@ -3,3 +3,4 @@ from .api import *
from .article import article_bp, articles_bp
from .feed import feed_bp, feeds_bp
+from .icon import icon_bp
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
index 530f3fef..7d8cdf38 100644
--- a/pyaggr3g470r/views/api/feed.py
+++ b/pyaggr3g470r/views/api/feed.py
@@ -20,7 +20,7 @@ FEED_ATTRS = {'title': {'type': str},
'site_link': {'type': str},
'enabled': {'type': bool, 'default': True},
'etag': {'type': str, 'default': ''},
- 'icon': {'type': str, 'default': ''},
+ 'icon_url': {'type': str, 'default': ''},
'last_modified': {'type': str},
'last_retrieved': {'type': str},
'last_error': {'type': str},
@@ -54,7 +54,7 @@ class FetchableFeedAPI(PyAggAbstractResource):
if g.user.refresh_rate:
args['refresh_rate'] = g.user.refresh_rate
- if args.pop('retreive_all'):
+ if args.pop('retreive_all', False):
contr = self.wider_controller
else:
contr = self.controller
diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py
index 6de07ad3..524bf6dd 100644
--- a/pyaggr3g470r/views/article.py
+++ b/pyaggr3g470r/views/article.py
@@ -4,6 +4,7 @@
from flask import Blueprint, g, render_template, redirect
from pyaggr3g470r import controllers, utils
+from pyaggr3g470r.lib.view_utils import etag_match
from pyaggr3g470r.decorators import pyagg_default_decorator
articles_bp = Blueprint('articles', __name__, url_prefix='/articles')
@@ -19,6 +20,7 @@ def redirect_to_article(article_id):
@article_bp.route('/<int:article_id>', methods=['GET'])
@pyagg_default_decorator
+@etag_match
def article(article_id=None):
"""
Presents the content of an article.
diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py
index fb3ea4c7..afb51903 100644
--- a/pyaggr3g470r/views/feed.py
+++ b/pyaggr3g470r/views/feed.py
@@ -14,6 +14,7 @@ from flask.ext.login import login_required
import conf
from pyaggr3g470r import utils
+from pyaggr3g470r.lib.view_utils import etag_match
from pyaggr3g470r.lib.feed_utils import construct_feed_from
from pyaggr3g470r.forms import AddFeedForm
from pyaggr3g470r.controllers import FeedController, ArticleController
@@ -24,6 +25,7 @@ feed_bp = Blueprint('feed', __name__, url_prefix='/feed')
@feeds_bp.route('/', methods=['GET'])
@login_required
+@etag_match
def feeds():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(g.user.id)
@@ -35,6 +37,7 @@ def feeds():
@feed_bp.route('/<int:feed_id>', methods=['GET'])
@login_required
+@etag_match
def feed(feed_id=None):
"Presents detailed information about a feed."
feed = FeedController(g.user.id).get(id=feed_id)
@@ -138,6 +141,7 @@ def update(action, feed_id=None):
@feed_bp.route('/create', methods=['GET'])
@feed_bp.route('/edit/<int:feed_id>', methods=['GET'])
@login_required
+@etag_match
def form(feed_id=None):
action = gettext("Add a feed")
head_titles = [action]
@@ -196,20 +200,3 @@ def process_form(feed_id=None):
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=new_feed.id))
-
-
-@feed_bp.route('/icon/<int:feed_id>', methods=['GET'])
-@login_required
-def icon(feed_id):
- icon = FeedController(None if g.user.is_admin() else g.user.id)\
- .get(id=feed_id).icon
- etag = md5(icon.encode('utf8')).hexdigest()
- headers = {'Cache-Control': 'max-age=86400', 'etag': etag}
- if request.headers.get('if-none-match') == etag:
- return Response(status=304, headers=headers)
- if '\n' in icon:
- content_type, *_, icon = icon.split()
- headers['content-type'] = content_type
- else:
- headers['content-type'] = 'application/image'
- return Response(base64.b64decode(icon), headers=headers)
diff --git a/pyaggr3g470r/views/icon.py b/pyaggr3g470r/views/icon.py
new file mode 100644
index 00000000..2f51304a
--- /dev/null
+++ b/pyaggr3g470r/views/icon.py
@@ -0,0 +1,14 @@
+import base64
+from flask import Blueprint, Response, request
+from pyaggr3g470r.controllers import IconController
+from pyaggr3g470r.lib.view_utils import etag_match
+
+icon_bp = Blueprint('icon', __name__, url_prefix='/icon')
+
+@icon_bp.route('/', methods=['GET'])
+@etag_match
+def icon():
+ icon = IconController().get(url=request.args['url'])
+ headers = {'Cache-Control': 'max-age=86400',
+ 'Content-Type': icon.mimetype}
+ return Response(base64.b64decode(icon.content), headers=headers)
diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py
index 0071f887..b649d5c6 100644
--- a/pyaggr3g470r/views/views.py
+++ b/pyaggr3g470r/views/views.py
@@ -36,7 +36,7 @@ from collections import OrderedDict
from bootstrap import application as app, db
from flask import render_template, request, flash, session, \
- url_for, redirect, g, current_app, make_response, Response
+ url_for, redirect, g, current_app, make_response
from flask.ext.login import LoginManager, login_user, logout_user, \
login_required, current_user, AnonymousUserMixin
from flask.ext.principal import Principal, Identity, AnonymousIdentity, \
@@ -48,8 +48,8 @@ from sqlalchemy.exc import IntegrityError
from werkzeug import generate_password_hash
import conf
-from pyaggr3g470r.lib.utils import to_hash
from pyaggr3g470r import utils, notifications, export
+from pyaggr3g470r.lib.view_utils import etag_match
from pyaggr3g470r.models import User, Feed, Article, Role
from pyaggr3g470r.decorators import feed_access_required
from pyaggr3g470r.forms import SignupForm, SigninForm, InformationMessageForm,\
@@ -229,6 +229,7 @@ def signup():
return render_template('signup.html', form=form)
+@etag_match
def render_home(filters=None, head_titles=None,
page_to_render='home', **kwargs):
if filters is None:
@@ -292,19 +293,12 @@ def render_home(filters=None, head_titles=None,
and filter_ != 'all' and not articles:
return redirect(gen_url(filter_='all'))
- etag = to_hash("".join([str(filters[key]) for key in sorted(filters)])
- + "".join([str(art.id) for art in articles]))
- if request.headers.get('if-none-match') == etag:
- return Response(status=304, headers={'etag': etag,
- 'Cache-Control': 'pragma: no-cache'})
- response = make_response(render_template('home.html', gen_url=gen_url,
- feed_id=feed_id, page_to_render=page_to_render,
- filter_=filter_, limit=limit, feeds=feeds,
- unread=arti_contr.count_by_feed(readed=False),
- articles=articles, in_error=in_error,
- head_titles=head_titles, sort_=sort_, **kwargs))
- response.headers['etag'] = etag
- return response
+ return render_template('home.html', gen_url=gen_url,
+ feed_id=feed_id, page_to_render=page_to_render,
+ filter_=filter_, limit=limit, feeds=feeds,
+ unread=arti_contr.count_by_feed(readed=False),
+ articles=articles, in_error=in_error,
+ head_titles=head_titles, sort_=sort_, **kwargs)
@app.route('/')
@@ -362,7 +356,9 @@ def fetch(feed_id=None):
"for administrator, on the Heroku platform."), "info")
return redirect(redirect_url())
+
@app.route('/about', methods=['GET'])
+@etag_match
def about():
"""
'About' page.
diff --git a/runserver.py b/runserver.py
index 5f20ddd4..ccd8bc60 100755
--- a/runserver.py
+++ b/runserver.py
@@ -51,6 +51,7 @@ with application.app_context():
application.register_blueprint(views.article_bp)
application.register_blueprint(views.feeds_bp)
application.register_blueprint(views.feed_bp)
+ application.register_blueprint(views.icon_bp)
if __name__ == '__main__':
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/__init__.py
diff --git a/scripts/probes.py b/scripts/probes.py
new file mode 100644
index 00000000..4c632184
--- /dev/null
+++ b/scripts/probes.py
@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+import sys
+from datetime import datetime, timedelta
+from flask.ext.script import Command, Option
+
+from pyaggr3g470r.controllers \
+ import UserController, FeedController, ArticleController
+DEFAULT_HEADERS = {'Content-Type': 'application/json', 'User-Agent': 'munin'}
+LATE_AFTER = 60
+FETCH_RATE = 3
+
+
+class AbstractMuninPlugin(Command):
+ urn = None
+
+ def execute(self):
+ raise NotImplementedError()
+
+ def config(self):
+ raise NotImplementedError()
+
+ def get_options(self):
+ if sys.argv[-1] == 'config':
+ return [Option(dest='config', default=sys.argv[-1] == 'config')]
+ return []
+
+ def run(self, config=False):
+ if config:
+ self.config()
+ else:
+ self.execute()
+
+
+class FeedProbe(AbstractMuninPlugin):
+
+ def config(self):
+ print("graph_title PyAgg - Feeds counts")
+ print("graph_vlabel feeds")
+ print("feeds.label Late feeds")
+ print("feeds_total.label Total feeds")
+ print("feeds.warning 15")
+ print("feeds.critical 30")
+ print("graph_category web")
+ print("graph_scale yes")
+
+ def execute(self):
+ delta = datetime.now() - timedelta(minutes=LATE_AFTER + FETCH_RATE + 1)
+ total = FeedController().read().count()
+
+ print("feeds.value %d"
+ % len(FeedController().list_late(delta, limit=total)))
+ print("feeds_total.value %d" % FeedController().read().count())
+
+
+class ArticleProbe(AbstractMuninPlugin):
+
+ def config(self):
+ print("graph_title Pyagg - Articles adding rate")
+ print("graph_vlabel Articles per sec")
+ print("articles.label Overall rate")
+ print("articles.type DERIVE")
+ print("articles.min 0")
+ for id_ in sorted(user.id for user in UserController().read()):
+ print("articles_user_%s.label Rate for user %s" % (id_, id_))
+ print("articles_user_%s.type DERIVE" % id_)
+ print("articles_user_%s.min 0" % id_)
+ print("graph_category web")
+ print("graph_scale yes")
+
+ def execute(self):
+ counts = ArticleController().count_by_user_id()
+ print("articles.value %s" % sum(counts.values()))
+ for user, count in counts.items():
+ print("articles_user_%s.value %s" % (user, count))
bgstack15