aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-03-04 07:48:14 +0100
committerCédric Bonhomme <kimble.mandel+bitbucket@gmail.com>2015-03-04 07:48:14 +0100
commit1c0ba8fad47489ce987e628605fd106f981d8075 (patch)
treeb381e4c8c910b2f2f282f6dadbda1a8c1938e358 /pyaggr3g470r
parentTypo. (diff)
parentadding refresh rate to the profile form (diff)
downloadnewspipe-1c0ba8fad47489ce987e628605fd106f981d8075.tar.gz
newspipe-1c0ba8fad47489ce987e628605fd106f981d8075.tar.bz2
newspipe-1c0ba8fad47489ce987e628605fd106f981d8075.zip
Merged in jaesivsm/pyaggr3g470r/evol/api (pull request #6)
Evolution Arch, API and new crawler
Diffstat (limited to 'pyaggr3g470r')
-rw-r--r--pyaggr3g470r/__init__.py48
-rw-r--r--pyaggr3g470r/controllers/__init__.py6
-rw-r--r--pyaggr3g470r/controllers/abstract.py69
-rw-r--r--pyaggr3g470r/controllers/article.py27
-rw-r--r--pyaggr3g470r/controllers/feed.py24
-rw-r--r--pyaggr3g470r/controllers/user.py7
-rw-r--r--pyaggr3g470r/decorators.py13
-rw-r--r--pyaggr3g470r/emails.py1
-rw-r--r--pyaggr3g470r/forms.py97
-rw-r--r--pyaggr3g470r/lib/__init__.py0
-rw-r--r--pyaggr3g470r/lib/crawler.py257
-rw-r--r--pyaggr3g470r/lib/utils.py14
-rw-r--r--pyaggr3g470r/models.py152
-rw-r--r--pyaggr3g470r/models/__init__.py35
-rw-r--r--pyaggr3g470r/models/article.py87
-rw-r--r--pyaggr3g470r/models/feed.py75
-rw-r--r--pyaggr3g470r/models/role.py41
-rw-r--r--pyaggr3g470r/models/user.py89
-rw-r--r--pyaggr3g470r/notifications.py3
-rw-r--r--pyaggr3g470r/rest.py351
-rw-r--r--pyaggr3g470r/search.py2
-rw-r--r--pyaggr3g470r/static/js/articles.js8
-rw-r--r--pyaggr3g470r/templates/edit_feed.html1
-rw-r--r--pyaggr3g470r/templates/feed.html10
-rw-r--r--pyaggr3g470r/templates/home.html6
-rw-r--r--pyaggr3g470r/templates/layout.html10
-rw-r--r--pyaggr3g470r/templates/profile.html5
-rwxr-xr-xpyaggr3g470r/utils.py9
-rw-r--r--pyaggr3g470r/views/__init__.py5
-rw-r--r--pyaggr3g470r/views/api/__init__.py31
-rw-r--r--pyaggr3g470r/views/api/article.py58
-rw-r--r--pyaggr3g470r/views/api/common.py217
-rw-r--r--pyaggr3g470r/views/api/feed.py57
-rw-r--r--pyaggr3g470r/views/article.py53
-rw-r--r--pyaggr3g470r/views/feed.py50
-rw-r--r--pyaggr3g470r/views/views.py (renamed from pyaggr3g470r/views.py)138
36 files changed, 1334 insertions, 722 deletions
diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py
index f3f784f4..e69de29b 100644
--- a/pyaggr3g470r/__init__.py
+++ b/pyaggr3g470r/__init__.py
@@ -1,48 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-from flask import Flask
-from flask.ext.sqlalchemy import SQLAlchemy
-from flask.ext.babel import Babel
-from flask.ext.babel import format_datetime
-
-import conf
-
-# Create Flask application
-app = Flask(__name__)
-app.debug = conf.WEBSERVER_DEBUG
-
-# Create dummy secrey key so we can use sessions
-app.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None)
-if not app.config['SECRET_KEY']:
- app.config['SECRET_KEY'] = os.urandom(12)
-app.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI
-db = SQLAlchemy(app)
-
-app.config['RECAPTCHA_USE_SSL'] = True
-app.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY
-app.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY
-
-if conf.ON_HEROKU:
- from flask_sslify import SSLify
- sslify = SSLify(app)
-
-ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json'])
-
-def allowed_file(filename):
- """
- Check if the uploaded file is allowed.
- """
- return '.' in filename and \
- filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
-
-babel = Babel(app)
-
-app.jinja_env.filters['datetime'] = format_datetime
-
-# Views
-from flask.ext.restful import Api
-api = Api(app)
-
-from pyaggr3g470r import views, rest
diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py
new file mode 100644
index 00000000..d8d1a104
--- /dev/null
+++ b/pyaggr3g470r/controllers/__init__.py
@@ -0,0 +1,6 @@
+from .feed import FeedController
+from .article import ArticleController
+from .user import UserController
+
+
+__all__ = ['FeedController', 'ArticleController', 'UserController']
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
new file mode 100644
index 00000000..a99e67f3
--- /dev/null
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -0,0 +1,69 @@
+import logging
+from bootstrap import db
+from sqlalchemy import update
+from werkzeug.exceptions import Forbidden, NotFound
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractController(object):
+ _db_cls = None # reference to the database class
+ _user_id_key = 'user_id'
+
+ def __init__(self, user_id):
+ self.user_id = user_id
+
+ def _to_filters(self, **filters):
+ if self.user_id:
+ filters[self._user_id_key] = self.user_id
+ db_filters = set()
+ for key, value in filters.items():
+ if key.endswith('__gt'):
+ db_filters.add(getattr(self._db_cls, key[:-4]) > value)
+ elif key.endswith('__lt'):
+ db_filters.add(getattr(self._db_cls, key[:-4]) < value)
+ elif key.endswith('__ge'):
+ db_filters.add(getattr(self._db_cls, key[:-4]) >= value)
+ elif key.endswith('__le'):
+ db_filters.add(getattr(self._db_cls, key[:-4]) <= value)
+ elif key.endswith('__ne'):
+ db_filters.add(getattr(self._db_cls, key[:-4]) != value)
+ elif key.endswith('__in'):
+ db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
+ else:
+ db_filters.add(getattr(self._db_cls, key) == value)
+ return db_filters
+
+ def _get(self, **filters):
+ return self._db_cls.query.filter(*self._to_filters(**filters))
+
+ def get(self, **filters):
+ obj = self._get(**filters).first()
+ if not obj:
+ raise NotFound({'message': 'No %r (%r)'
+ % (self._db_cls.__class__.__name__, filters)})
+ if getattr(obj, self._user_id_key) != self.user_id:
+ raise Forbidden({'message': 'No authorized to access %r (%r)'
+ % (self._db_cls.__class__.__name__, filters)})
+ return obj
+
+ def create(self, **attrs):
+ attrs[self._user_id_key] = self.user_id
+ obj = self._db_cls(**attrs)
+ db.session.add(obj)
+ db.session.commit()
+ return obj
+
+ def read(self, **filters):
+ return self._get(**filters)
+
+ def update(self, filters, attrs):
+ result = self._get(**filters).update(attrs, synchronize_session=False)
+ db.session.commit()
+ return result
+
+ def delete(self, obj_id):
+ obj = self.get(id=obj_id)
+ db.session.delete(obj)
+ db.session.commit()
+ return obj
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
new file mode 100644
index 00000000..46ca0988
--- /dev/null
+++ b/pyaggr3g470r/controllers/article.py
@@ -0,0 +1,27 @@
+import conf
+from .abstract import AbstractController
+from pyaggr3g470r.models import Article
+
+
+class ArticleController(AbstractController):
+ _db_cls = Article
+
+ def get(self, **filters):
+ article = super(ArticleController, self).get(**filters)
+ if not article.readed:
+ self.update({'id': article.id}, {'readed': True})
+ return article
+
+ def delete(self, obj_id):
+ obj = super(ArticleController, self).delete(obj_id)
+ if not conf.ON_HEROKU:
+ import pyaggr3g470r.search as fastsearch
+ fastsearch.delete_article(self.user_id, obj.feed_id, obj_id)
+ return obj
+
+ def challenge(self, ids):
+ """Will return each id that wasn't found in the database."""
+ for id_ in ids:
+ if self.read(**id_).first():
+ continue
+ yield id_
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
new file mode 100644
index 00000000..a2455e2b
--- /dev/null
+++ b/pyaggr3g470r/controllers/feed.py
@@ -0,0 +1,24 @@
+from datetime import datetime, timedelta
+from .abstract import AbstractController
+from pyaggr3g470r.models import Feed
+
+DEFAULT_MAX_ERROR = 3
+DEFAULT_LIMIT = 5
+
+
+class FeedController(AbstractController):
+ _db_cls = Feed
+
+ def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
+ from pyaggr3g470r.controllers import UserController
+ now = datetime.now()
+ user = UserController(self.user_id).get(id=self.user_id)
+ max_last = now - timedelta(minutes=user.refresh_rate or 60)
+ feeds = [feed for feed in self.read(user_id=self.user_id,
+ error_count__lt=max_error, enabled=True,
+ last_retreived__lt=max_last).limit(limit)]
+
+ if feeds:
+ self.update({'id__in': [feed.id for feed in feeds]},
+ {'last_retreived': now})
+ return feeds
diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py
new file mode 100644
index 00000000..c6c1d545
--- /dev/null
+++ b/pyaggr3g470r/controllers/user.py
@@ -0,0 +1,7 @@
+from .abstract import AbstractController
+from pyaggr3g470r.models import User
+
+
+class UserController(AbstractController):
+ _db_cls = User
+ _user_id_key = 'id'
diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py
index 3e808793..9e8f9c0b 100644
--- a/pyaggr3g470r/decorators.py
+++ b/pyaggr3g470r/decorators.py
@@ -1,10 +1,12 @@
#! /usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
from threading import Thread
from functools import wraps
from flask import g, redirect, url_for, flash
+from flask.ext.babel import gettext
+from flask.ext.login import login_required
from pyaggr3g470r.models import Feed
@@ -20,6 +22,7 @@ def async(f):
thr.start()
return wrapper
+
def feed_access_required(func):
"""
This decorator enables to check if a user has access to a feed.
@@ -37,3 +40,11 @@ def feed_access_required(func):
else:
return func(*args, **kwargs)
return decorated
+
+
+def pyagg_default_decorator(func):
+ @login_required
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ return func(*args, **kwargs)
+ return wrapper
diff --git a/pyaggr3g470r/emails.py b/pyaggr3g470r/emails.py
index 50f94761..9a129718 100644
--- a/pyaggr3g470r/emails.py
+++ b/pyaggr3g470r/emails.py
@@ -31,6 +31,7 @@ from pyaggr3g470r.decorators import async
logger = logging.getLogger(__name__)
+
@async
def send_async_email(mfrom, mto, msg):
try:
diff --git a/pyaggr3g470r/forms.py b/pyaggr3g470r/forms.py
index 58abb864..e385c4e2 100644
--- a/pyaggr3g470r/forms.py
+++ b/pyaggr3g470r/forms.py
@@ -29,47 +29,52 @@ __license__ = "GPLv3"
from flask import flash
from flask.ext.wtf import Form
from flask.ext.babel import lazy_gettext
-from wtforms import TextField, TextAreaField, PasswordField, BooleanField, SubmitField, validators
+from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \
+ SubmitField, IntegerField, validators
from flask.ext.wtf.html5 import EmailField
from flask_wtf import RecaptchaField
from pyaggr3g470r.models import User
class SignupForm(Form):
- nickname = TextField(lazy_gettext("Nickname"), [validators.Required(lazy_gettext("Please enter your nickname."))])
- email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
- password = PasswordField(lazy_gettext("Password"), [validators.Required(lazy_gettext("Please enter a password.")), validators.Length(min=6, max=100)])
+ nickname = TextField(lazy_gettext("Nickname"),
+ [validators.Required(lazy_gettext("Please enter your nickname."))])
+ email = EmailField(lazy_gettext("Email"),
+ [validators.Length(min=6, max=35),
+ validators.Required(
+ lazy_gettext("Please enter your email address."))])
+ password = PasswordField(lazy_gettext("Password"),
+ [validators.Required(lazy_gettext("Please enter a password.")),
+ validators.Length(min=6, max=100)])
recaptcha = RecaptchaField()
submit = SubmitField(lazy_gettext("Sign up"))
- def __init__(self, *args, **kwargs):
- Form.__init__(self, *args, **kwargs)
-
def validate(self):
- if not Form.validate(self):
- return False
+ validated = super(SignupForm, self).validate()
if self.nickname.data != User.make_valid_nickname(self.nickname.data):
self.nickname.errors.append(lazy_gettext('This nickname has invalid characters. Please use letters, numbers, dots and underscores only.'))
- return False
- return True
+ validated = False
+ return validated
+
class SigninForm(Form):
"""
Sign in form.
"""
- email = EmailField("Email", [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
- password = PasswordField(lazy_gettext('Password'), [validators.Required(lazy_gettext("Please enter a password.")), validators.Length(min=6, max=100)])
+ email = EmailField("Email", [validators.Length(min=6, max=35),
+ validators.Required(lazy_gettext("Please enter your email address."))])
+ password = PasswordField(lazy_gettext('Password'),
+ [validators.Required(lazy_gettext("Please enter a password.")),
+ validators.Length(min=6, max=100)])
submit = SubmitField(lazy_gettext("Log In"))
- def __init__(self, *args, **kwargs):
- Form.__init__(self, *args, **kwargs)
-
def validate(self):
- if not Form.validate(self):
+ if not super(SigninForm, self).validate():
return False
user = User.query.filter(User.email == self.email.data).first()
- if user and user.check_password(self.password.data) and user.activation_key == "":
+ if user and user.check_password(self.password.data) \
+ and user.activation_key == "":
return True
elif user and user.activation_key != "":
flash(lazy_gettext('Account not confirmed'), 'danger')
@@ -79,6 +84,7 @@ class SigninForm(Form):
#self.email.errors.append("Invalid email or password")
return False
+
class AddFeedForm(Form):
title = TextField(lazy_gettext("Title"), [validators.Optional()])
link = TextField(lazy_gettext("Feed link"), [validators.Optional()])
@@ -86,45 +92,52 @@ class AddFeedForm(Form):
enabled = BooleanField(lazy_gettext("Check for updates"), default=True)
submit = SubmitField(lazy_gettext("Save"))
- def __init__(self, *args, **kwargs):
- Form.__init__(self, *args, **kwargs)
-
- def validate(self):
- if not Form.validate(self):
- return False
- return True
class ProfileForm(Form):
- nickname = TextField(lazy_gettext("Nickname"), [validators.Required(lazy_gettext("Please enter your nickname."))])
- email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email."))])
+ nickname = TextField(lazy_gettext("Nickname"),
+ [validators.Required(lazy_gettext("Please enter your nickname."))])
+ email = EmailField(lazy_gettext("Email"),
+ [validators.Length(min=6, max=35),
+ validators.Required(lazy_gettext("Please enter your email."))])
password = PasswordField(lazy_gettext("Password"))
+ password_conf = PasswordField(lazy_gettext("Password Confirmation"))
+ refresh_rate = IntegerField(lazy_gettext("Feeds refresh frequency "
+ "(in minutes)"),
+ default=60)
submit = SubmitField(lazy_gettext("Save"))
- def __init__(self, *args, **kwargs):
- Form.__init__(self, *args, **kwargs)
-
def validate(self):
- if not Form.validate(self):
- return False
+ validated = super(ProfileForm, self).validate()
+ if self.password.data != self.password_conf.data:
+ message = lazy_gettext("Passwords aren't the same.")
+ self.password.errors.append(message)
+ self.password_conf.errors.append(message)
+ validated = False
if self.nickname.data != User.make_valid_nickname(self.nickname.data):
- self.nickname.errors.append(lazy_gettext('This nickname has invalid characters. Please use letters, numbers, dots and underscores only.'))
- return False
- return True
+ self.nickname.errors.append(lazy_gettext('This nickname has '
+ 'invalid characters. Please use letters, numbers, dots and'
+ ' underscores only.'))
+ validated = False
+ return validated
+
class InformationMessageForm(Form):
- subject = TextField(lazy_gettext("Subject"), [validators.Required(lazy_gettext("Please enter a subject."))])
- message = TextAreaField(lazy_gettext("Message"), [validators.Required(lazy_gettext("Please enter a content."))])
+ subject = TextField(lazy_gettext("Subject"),
+ [validators.Required(lazy_gettext("Please enter a subject."))])
+ message = TextAreaField(lazy_gettext("Message"),
+ [validators.Required(lazy_gettext("Please enter a content."))])
submit = SubmitField(lazy_gettext("Send"))
+
class RecoverPasswordForm(Form):
- email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
+ email = EmailField(lazy_gettext("Email"),
+ [validators.Length(min=6, max=35),
+ validators.Required(
+ lazy_gettext("Please enter your email address."))])
submit = SubmitField(lazy_gettext("Recover"))
- def __init__(self, *args, **kwargs):
- Form.__init__(self, *args, **kwargs)
-
def validate(self):
- if not Form.validate(self):
+ if not super(RecoverPasswordForm, self).validate():
return False
user = User.query.filter(User.email == self.email.data).first()
diff --git a/pyaggr3g470r/lib/__init__.py b/pyaggr3g470r/lib/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/pyaggr3g470r/lib/__init__.py
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
new file mode 100644
index 00000000..64ef8b6d
--- /dev/null
+++ b/pyaggr3g470r/lib/crawler.py
@@ -0,0 +1,257 @@
+"""
+Here's a sum up on how it works :
+
+CrawlerScheduler.run
+ will retreive a list of feeds to be refreshed and pass result to
+CrawlerScheduler.callback
+ which will retreive each feed and treat result with
+FeedCrawler.callback
+ which will interprete the result (status_code, etag) collect ids
+ and match them agaisnt pyagg which will cause
+PyAggUpdater.callback
+ to create the missing entries
+"""
+
+import time
+import conf
+import json
+import logging
+import requests
+import feedparser
+import dateutil.parser
+from functools import wraps
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor
+from requests_futures.sessions import FuturesSession
+from pyaggr3g470r.lib.utils import default_handler
+
+logger = logging.getLogger(__name__)
+API_ROOT = "api/v2.0/"
+
+
+def extract_id(entry, keys=[('link', 'link'),
+ ('published', 'retrieved_date'),
+ ('updated', 'retrieved_date')], force_id=False):
+ """For a given entry will return a dict that allows to identify it. The
+ dict will be constructed on the uid of the entry. if that identifier is
+ absent, the dict will be constructed upon the values of "keys".
+ """
+ entry_id = entry.get('entry_id') or entry.get('id')
+ if entry_id:
+ return {'entry_id': entry_id}
+ if not entry_id and force_id:
+ entry_id = hash("".join(entry[entry_key] for _, entry_key in keys
+ if entry_key in entry))
+ else:
+ ids = {}
+ for entry_key, pyagg_key in keys:
+ if entry_key in entry and pyagg_key not in ids:
+ ids[pyagg_key] = entry[entry_key]
+ if 'date' in pyagg_key:
+ ids[pyagg_key] = dateutil.parser.parse(ids[pyagg_key])\
+ .isoformat()
+ return ids
+
+
+class AbstractCrawler:
+ __session__ = None
+ __counter__ = 0
+
+ def __init__(self, auth):
+ self.auth = auth
+ self.session = self.get_session()
+ self.url = conf.PLATFORM_URL
+
+ @classmethod
+ def get_session(cls):
+ """methods that allows us to treat session as a singleton"""
+ if cls.__session__ is None:
+ cls.__session__ = FuturesSession(
+ executor=ThreadPoolExecutor(max_workers=conf.NB_WORKER))
+ cls.__session__.verify = False
+ return cls.__session__
+
+ @classmethod
+ def count_on_me(cls, func):
+ """A basic decorator which will count +1 at the begining of a call
+ and -1 at the end. It kinda allows us to wait for the __counter__ value
+ to be 0, meaning nothing is done anymore."""
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ cls.__counter__ += 1
+ result = func(*args, **kwargs)
+ cls.__counter__ -= 1
+ return result
+ return wrapper
+
+ def query_pyagg(self, method, urn, data=None):
+ """A wrapper for internal call, method should be ones you can find
+ on requests (header, post, get, options, ...), urn the distant
+ resources you want to access on pyagg, and data, the data you wanna
+ transmit."""
+ if data is None:
+ data = {}
+ method = getattr(self.session, method)
+ return method("%s%s%s" % (self.url, API_ROOT, urn),
+ auth=self.auth, data=json.dumps(data,
+ default=default_handler),
+ headers={'Content-Type': 'application/json'})
+
+ @classmethod
+ def wait(cls):
+ "See count_on_me, that method will just wait for the counter to be 0"
+ time.sleep(1)
+ while cls.__counter__:
+ time.sleep(1)
+
+
+class PyAggUpdater(AbstractCrawler):
+
+ def __init__(self, feed, entries, headers, auth):
+ self.feed = feed
+ self.entries = entries
+ self.headers = headers
+ super(PyAggUpdater, self).__init__(auth)
+
+ def to_article(self, entry):
+ "Safe method to transorm a feedparser entry into an article"
+ date = datetime.now()
+
+ for date_key in ('published', 'updated'):
+ if entry.get(date_key):
+ try:
+ date = dateutil.parser.parse(entry[date_key])
+ except Exception:
+ pass
+ else:
+ break
+ content = ''
+ if entry.get('content'):
+ content = entry['content'][0]['value']
+ elif entry.get('summary'):
+ content = entry['summary']
+
+ return {'feed_id': self.feed['id'],
+ 'entry_id': extract_id(entry).get('entry_id', None),
+ 'link': entry.get('link', self.feed['site_link']),
+ 'title': entry.get('title', 'No title'),
+ 'readed': False, 'like': False,
+ 'content': content,
+ 'retrieved_date': date.isoformat(),
+ 'date': date.isoformat()}
+
+ @AbstractCrawler.count_on_me
+ def callback(self, response):
+ """Will process the result from the challenge, creating missing article
+ and updating the feed"""
+ results = response.result().json()
+ logger.debug('%r %r - %d entries were not matched and will be created',
+ self.feed['id'], self.feed['title'], len(results))
+ for id_to_create in results:
+ entry = self.to_article(
+ self.entries[tuple(sorted(id_to_create.items()))])
+ logger.info('creating %r - %r', entry['title'], id_to_create)
+ self.query_pyagg('post', 'article', entry)
+
+ now = datetime.now()
+ logger.debug('%r %r - updating feed etag %r last_mod %r',
+ self.feed['id'], self.feed['title'],
+ self.headers.get('etag'), now)
+
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': 0,
+ 'etag': self.headers.get('etag', ''),
+ 'last_error': '',
+ 'last_modified': self.headers.get('last-modified', '')})
+
+
+class FeedCrawler(AbstractCrawler):
+
+ def __init__(self, feed, auth):
+ self.feed = feed
+ super(FeedCrawler, self).__init__(auth)
+
+ def clean_feed(self):
+ """Will reset the errors counters on a feed that have known errors"""
+ if self.feed.get('error_count') or self.feed.get('last_error'):
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+ {'error_count': 0, 'last_error': ''})
+
+ @AbstractCrawler.count_on_me
+ def callback(self, response):
+ """will fetch the feed and interprete results (304, etag) or will
+ challenge pyagg to compare gotten entries with existing ones"""
+ try:
+ response = response.result()
+ response.raise_for_status()
+ except Exception as error:
+ error_count = self.feed['error_count'] + 1
+ logger.warn('%r %r - an error occured while fetching feed; bumping'
+ ' error count to %r', self.feed['id'],
+ self.feed['title'], error_count)
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+ {'error_count': error_count,
+ 'last_error': str(error)})
+ return
+
+ if response.status_code == 304:
+ logger.info("%r %r - feed responded with 304",
+ self.feed['id'], self.feed['title'])
+ self.clean_feed()
+ return
+ if self.feed['etag'] and response.headers.get('etag') \
+ and response.headers.get('etag') == self.feed['etag']:
+ logger.info("%r %r - feed responded with same etag (%d)",
+ self.feed['id'], self.feed['title'],
+ response.status_code)
+ self.clean_feed()
+ return
+ ids, entries = [], {}
+ parsed_response = feedparser.parse(response.text)
+ for entry in parsed_response['entries']:
+ entries[tuple(sorted(extract_id(entry).items()))] = entry
+ ids.append(extract_id(entry))
+ logger.debug('%r %r - found %d entries %r',
+ self.feed['id'], self.feed['title'], len(ids), ids)
+ future = self.query_pyagg('get', 'articles/challenge', {'ids': ids})
+ updater = PyAggUpdater(self.feed, entries, response.headers, self.auth)
+ future.add_done_callback(updater.callback)
+
+
+class CrawlerScheduler(AbstractCrawler):
+
+ def __init__(self, username, password):
+ self.auth = (username, password)
+ super(CrawlerScheduler, self).__init__(self.auth)
+
+ def prepare_headers(self, feed):
+ """For a known feed, will construct some header dictionnary"""
+ headers = {}
+ if feed.get('etag', None):
+ headers['If-None-Match'] = feed['etag']
+ if feed.get('last_modified'):
+ headers['If-Modified-Since'] = feed['last_modified']
+ logger.debug('%r %r - calculated headers %r',
+ feed['id'], feed['title'], headers)
+ return headers
+
+ @AbstractCrawler.count_on_me
+ def callback(self, response):
+ """processes feeds that need to be fetched"""
+ response = response.result()
+ response.raise_for_status()
+ feeds = response.json()
+ logger.debug('%d to fetch %r', len(feeds), feeds)
+ for feed in feeds:
+ logger.info('%r %r - fetching resources',
+ feed['id'], feed['title'])
+ future = self.session.get(feed['link'],
+ headers=self.prepare_headers(feed))
+ future.add_done_callback(FeedCrawler(feed, self.auth).callback)
+
+ @AbstractCrawler.count_on_me
+ def run(self, **kwargs):
+ """entry point, will retreive feeds to be fetch
+ and launch the whole thing"""
+ logger.debug('retreving fetchable feed')
+ future = self.query_pyagg('get', 'feeds/fetchable', kwargs)
+ future.add_done_callback(self.callback)
diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py
new file mode 100644
index 00000000..a4f4b3ec
--- /dev/null
+++ b/pyaggr3g470r/lib/utils.py
@@ -0,0 +1,14 @@
+import types
+
+def default_handler(obj):
+ """JSON handler for default query formatting"""
+ if hasattr(obj, 'isoformat'):
+ return obj.isoformat()
+ if hasattr(obj, 'dump'):
+ return obj.dump()
+ if isinstance(obj, (set, frozenset, types.GeneratorType)):
+ return list(obj)
+ if isinstance(obj, BaseException):
+ return str(obj)
+ raise TypeError("Object of type %s with value of %r "
+ "is not JSON serializable" % (type(obj), obj))
diff --git a/pyaggr3g470r/models.py b/pyaggr3g470r/models.py
deleted file mode 100644
index 8bc9e76b..00000000
--- a/pyaggr3g470r/models.py
+++ /dev/null
@@ -1,152 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# pyAggr3g470r - A Web based news aggregator.
-# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
-#
-# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 0.4 $"
-__date__ = "$Date: 2013/11/05 $"
-__revision__ = "$Date: 2014/04/12 $"
-__copyright__ = "Copyright (c) Cedric Bonhomme"
-__license__ = "GPLv3"
-
-import re
-import json
-import random, hashlib
-from datetime import datetime
-from sqlalchemy import asc, desc
-from werkzeug import generate_password_hash, check_password_hash
-from flask.ext.login import UserMixin
-
-from pyaggr3g470r import db
-
-class User(db.Model, UserMixin):
- """
- Represent a user.
- """
- id = db.Column(db.Integer, primary_key = True)
- nickname = db.Column(db.String(), unique = True)
- email = db.Column(db.String(254), index = True, unique = True)
- pwdhash = db.Column(db.String())
- roles = db.relationship('Role', backref = 'user', lazy = 'dynamic')
- activation_key = db.Column(db.String(128), default =
- hashlib.sha512(
- str(random.getrandbits(256)).encode("utf-8")
- ).hexdigest()[:86])
- date_created = db.Column(db.DateTime(), default=datetime.now)
- last_seen = db.Column(db.DateTime(), default=datetime.now)
- feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan')
-
- @staticmethod
- def make_valid_nickname(nickname):
- return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
-
- def get_id(self):
- """
- Return the id (email) of the user.
- """
- return self.email
-
- def set_password(self, password):
- """
- Hash the password of the user.
- """
- self.pwdhash = generate_password_hash(password)
-
- def check_password(self, password):
- """
- Check the password of the user.
- """
- return check_password_hash(self.pwdhash, password)
-
- def is_admin(self):
- """
- Return True if the user has administrator rights.
- """
- return len([role for role in self.roles if role.name == "admin"]) != 0
-
- def __eq__(self, other):
- return self.id == other.id
-
- def __repr__(self):
- return '<User %r>' % (self.nickname)
-
-class Role(db.Model):
- """
- Represent a role.
- """
- id = db.Column(db.Integer, primary_key = True)
- name = db.Column(db.String(), unique = True)
-
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-
-class Feed(db.Model):
- """
- Represent a feed.
- """
- id = db.Column(db.Integer, primary_key = True)
- title = db.Column(db.String(), default="No title")
- description = db.Column(db.String(), default="FR")
- link = db.Column(db.String())
- site_link = db.Column(db.String(), default="")
- enabled = db.Column(db.Boolean(), default=True)
- created_date = db.Column(db.DateTime(), default=datetime.now)
- articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan',
- order_by=desc("Article.date"))
-
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-
- def __repr__(self):
- return '<Feed %r>' % (self.title)
-
-class Article(db.Model):
- """
- Represent an article from a feed.
- """
- id = db.Column(db.Integer, primary_key = True)
- entry_id = db.Column(db.String())
- link = db.Column(db.String())
- title = db.Column(db.String())
- content = db.Column(db.String())
- readed = db.Column(db.Boolean(), default=False)
- like = db.Column(db.Boolean(), default=False)
- date = db.Column(db.DateTime(), default=datetime.now)
- retrieved_date = db.Column(db.DateTime(), default=datetime.now)
-
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
- feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
-
- def previous_article(self):
- """
- Returns the previous article (older).
- """
- return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first()
-
- def next_article(self):
- """
- Returns the next article (newer).
- """
- return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first()
-
- def __repr__(self):
- return json.dumps({
- "title": self.title,
- "link": self.link,
- "content": self.content
- })
diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py
new file mode 100644
index 00000000..9584d1f2
--- /dev/null
+++ b/pyaggr3g470r/models/__init__.py
@@ -0,0 +1,35 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from .feed import Feed
+from .role import Role
+from .user import User
+from .article import Article
+
+
+__all__ = ['Feed', 'Role', 'User', 'Article']
diff --git a/pyaggr3g470r/models/article.py b/pyaggr3g470r/models/article.py
new file mode 100644
index 00000000..0466bc35
--- /dev/null
+++ b/pyaggr3g470r/models/article.py
@@ -0,0 +1,87 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+import json
+from datetime import datetime
+from flask import g
+from sqlalchemy import asc, desc
+
+db = g.db
+
+
+class Article(db.Model):
+ """
+ Represent an article from a feed.
+ """
+ id = db.Column(db.Integer, primary_key=True)
+ entry_id = db.Column(db.String())
+ link = db.Column(db.String())
+ title = db.Column(db.String())
+ content = db.Column(db.String())
+ readed = db.Column(db.Boolean(), default=False)
+ like = db.Column(db.Boolean(), default=False)
+ date = db.Column(db.DateTime(), default=datetime.now)
+ retrieved_date = db.Column(db.DateTime(), default=datetime.now)
+
+ user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+ feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
+
+ def previous_article(self):
+ """
+ Returns the previous article (older).
+ """
+ return Article.query.filter(Article.date < self.date,
+ Article.feed_id == self.feed_id)\
+ .order_by(desc("Article.date")).first()
+
+ def next_article(self):
+ """
+ Returns the next article (newer).
+ """
+ return Article.query.filter(Article.date > self.date,
+ Article.feed_id == self.feed_id)\
+ .order_by(asc("Article.date")).first()
+
+ def __repr__(self):
+ return json.dumps({
+ "title": self.title,
+ "link": self.link,
+ "content": self.content
+ })
+
+ def dump(self):
+ return {"id": self.id,
+ "title": self.title,
+ "link": self.link,
+ "content": self.content,
+ "readed": self.readed,
+ "like": self.like,
+ "date": self.date,
+ "retrieved_date": self.retrieved_date,
+ "feed_id": getattr(self.source, 'id', None),
+ "feed_name": getattr(self.source, 'title', None)}
diff --git a/pyaggr3g470r/models/feed.py b/pyaggr3g470r/models/feed.py
new file mode 100644
index 00000000..28cb2b92
--- /dev/null
+++ b/pyaggr3g470r/models/feed.py
@@ -0,0 +1,75 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from datetime import datetime
+from flask import g
+from sqlalchemy import desc
+
+db = g.db
+
+
+class Feed(db.Model):
+ """
+ Represent a station.
+ """
+ id = db.Column(db.Integer, primary_key=True)
+ title = db.Column(db.String(), default="No title")
+ description = db.Column(db.String(), default="FR")
+ link = db.Column(db.String())
+ site_link = db.Column(db.String(), default="")
+ enabled = db.Column(db.Boolean(), default=True)
+ created_date = db.Column(db.DateTime(), default=datetime.now)
+
+ # cache handling
+ etag = db.Column(db.String(), default="")
+ last_modified = db.Column(db.String(), default="")
+ last_retreived = db.Column(db.DateTime(), default=datetime(1970, 1, 1))
+
+ # error logging
+ last_error = db.Column(db.String(), default="")
+ error_count = db.Column(db.Integer(), default=0)
+
+ # relationship
+ user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+ articles = db.relationship('Article', backref='source', lazy='dynamic',
+ cascade='all,delete-orphan',
+ order_by=desc("Article.date"))
+
+ def __repr__(self):
+ return '<Feed %r>' % (self.title)
+
+ def dump(self):
+ return {"id": self.id,
+ "title": self.title,
+ "description": self.description,
+ "link": self.link,
+ "site_link": self.site_link,
+ "etag": self.etag,
+ "error_count": self.error_count,
+ "last_modified": self.last_modified,
+ "last_retreived": self.last_retreived}
diff --git a/pyaggr3g470r/models/role.py b/pyaggr3g470r/models/role.py
new file mode 100644
index 00000000..71497caf
--- /dev/null
+++ b/pyaggr3g470r/models/role.py
@@ -0,0 +1,41 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from flask import g
+
+db = g.db
+
+
+class Role(db.Model):
+ """
+ Represent a role.
+ """
+ id = db.Column(db.Integer, primary_key=True)
+ name = db.Column(db.String(), unique=True)
+
+ user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
diff --git a/pyaggr3g470r/models/user.py b/pyaggr3g470r/models/user.py
new file mode 100644
index 00000000..f2a268db
--- /dev/null
+++ b/pyaggr3g470r/models/user.py
@@ -0,0 +1,89 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+import re
+import random
+import hashlib
+from datetime import datetime
+from flask import g
+from werkzeug import generate_password_hash, check_password_hash
+from flask.ext.login import UserMixin
+
+db = g.db
+
+
+class User(db.Model, UserMixin):
+ """
+ Represent a user.
+ """
+ id = db.Column(db.Integer, primary_key=True)
+ nickname = db.Column(db.String(), unique=True)
+ email = db.Column(db.String(254), index=True, unique=True)
+ pwdhash = db.Column(db.String())
+ roles = db.relationship('Role', backref='user', lazy='dynamic')
+ activation_key = db.Column(db.String(128), default=hashlib.sha512(
+ str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86])
+ date_created = db.Column(db.DateTime(), default=datetime.now)
+ last_seen = db.Column(db.DateTime(), default=datetime.now)
+ feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic',
+ cascade='all,delete-orphan')
+ refresh_rate = db.Column(db.Integer, default=60) # in minutes
+
+ @staticmethod
+ def make_valid_nickname(nickname):
+ return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
+
+ def get_id(self):
+ """
+ Return the id (email) of the user.
+ """
+ return self.email
+
+ def set_password(self, password):
+ """
+ Hash the password of the user.
+ """
+ self.pwdhash = generate_password_hash(password)
+
+ def check_password(self, password):
+ """
+ Check the password of the user.
+ """
+ return check_password_hash(self.pwdhash, password)
+
+ def is_admin(self):
+ """
+ Return True if the user has administrator rights.
+ """
+ return len([role for role in self.roles if role.name == "admin"]) != 0
+
+ def __eq__(self, other):
+ return self.id == other.id
+
+ def __repr__(self):
+ return '<User %r>' % (self.nickname)
diff --git a/pyaggr3g470r/notifications.py b/pyaggr3g470r/notifications.py
index 1acf782e..cf8fb723 100644
--- a/pyaggr3g470r/notifications.py
+++ b/pyaggr3g470r/notifications.py
@@ -19,9 +19,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-from pyaggr3g470r import conf
+import conf
from pyaggr3g470r import emails
+
def information_message(subject, plaintext):
"""
Send an information message to the users of the platform.
diff --git a/pyaggr3g470r/rest.py b/pyaggr3g470r/rest.py
deleted file mode 100644
index 1f354167..00000000
--- a/pyaggr3g470r/rest.py
+++ /dev/null
@@ -1,351 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# pyAggr3g470r - A Web based news aggregator.
-# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/
-#
-# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 0.2 $"
-__date__ = "$Date: 2014/06/18 $"
-__revision__ = "$Date: 2014/07/05 $"
-__copyright__ = "Copyright (c) Cedric Bonhomme"
-__license__ = "AGPLv3"
-
-import re
-import dateutil.parser
-from functools import wraps
-from flask import g, Response, request, session, jsonify
-from flask.ext.restful import Resource, reqparse
-
-import conf
-if not conf.ON_HEROKU:
- import pyaggr3g470r.search as fastsearch
-from pyaggr3g470r import api, db
-from pyaggr3g470r.models import User, Article, Feed
-
-def authenticate(func):
- """
- Decorator for the authentication to the web services.
- """
- @wraps(func)
- def wrapper(*args, **kwargs):
- if not getattr(func, 'authenticated', True):
- return func(*args, **kwargs)
-
- # authentication based on the session (already logged on the site)
- if 'email' in session or g.user.is_authenticated():
- return func(*args, **kwargs)
-
- # authentication via HTTP only
- auth = request.authorization
- try:
- email = auth.username
- user = User.query.filter(User.email == email).first()
- if user and user.check_password(auth.password) and user.activation_key == "":
- g.user = user
- return func(*args, **kwargs)
- except AttributeError:
- pass
-
- return Response('<Authentication required>', 401,
- {'WWWAuthenticate':'Basic realm="Login Required"'})
- return wrapper
-
-class ArticleListAPI(Resource):
- """
- Defines a RESTful API for Article elements.
- """
- method_decorators = [authenticate]
-
- def __init__(self):
- self.reqparse = reqparse.RequestParser()
- self.reqparse.add_argument('title', type = unicode, location = 'json')
- self.reqparse.add_argument('content', type = unicode, location = 'json')
- self.reqparse.add_argument('link', type = unicode, location = 'json')
- self.reqparse.add_argument('date', type = str, location = 'json')
- self.reqparse.add_argument('feed_id', type = int, location = 'json')
- super(ArticleListAPI, self).__init__()
-
- def get(self):
- """
- Returns a list of articles.
- """
- feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled}
- articles = Article.query.filter(Article.feed_id.in_(feeds.keys()),
- Article.user_id == g.user.id)
- filter_ = request.args.get('filter_', 'unread')
- feed_id = int(request.args.get('feed', 0))
- limit = request.args.get('limit', 1000)
- if filter_ != 'all':
- articles = articles.filter(Article.readed == (filter_ == 'read'))
- if feed_id:
- articles = articles.filter(Article.feed_id == feed_id)
-
- articles = articles.order_by(Article.date.desc())
- if limit != 'all':
- limit = int(limit)
- articles = articles.limit(limit)
-
- return jsonify(result= [{
- "id": article.id,
- "title": article.title,
- "link": article.link,
- "content": article.content,
- "readed": article.readed,
- "like": article.like,
- "date": article.date,
- "retrieved_date": article.retrieved_date,
- "feed_id": article.source.id,
- "feed_name": article.source.title
- }
- for article in articles]
- )
-
- def post(self):
- """
- POST method - Create a new article.
- """
- args = self.reqparse.parse_args()
- article_dict = {}
- for k, v in args.iteritems():
- if v != None:
- article_dict[k] = v
- else:
- return {"message":"Missing argument: %s." % (k,)}
- article_date = None
- try:
- article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True)
- except:
- try: # trying to clean date field from letters
- article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True))
- except:
- return jsonify({"message":"Bad format for the date."})
- article = Article(link=article_dict["link"], title=article_dict["title"],
- content=article_dict["content"], readed=False, like=False,
- date=article_date, user_id=g.user.id,
- feed_id=article_dict["feed_id"])
- feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first()
- feed.articles.append(article)
- try:
- db.session.commit()
- return jsonify({"message":"ok"})
- except:
- return jsonify({"message":"Impossible to create the article."})
-
-class ArticleAPI(Resource):
- """
- Defines a RESTful API for Article elements.
- """
- method_decorators = [authenticate]
-
- def __init__(self):
- self.reqparse = reqparse.RequestParser()
- self.reqparse.add_argument('like', type = bool, location = 'json')
- self.reqparse.add_argument('readed', type = bool, location = 'json')
- super(ArticleAPI, self).__init__()
-
- def get(self, id=None):
- """
- Returns an article.
- """
- result = []
- if id is not None:
- article = Article.query.filter(Article.user_id == g.user.id, Article.id == id).first()
- if article is not None:
- if not article.readed:
- article.readed = True
- db.session.commit()
- result.append(article)
-
- return jsonify(result= [{
- "id": article.id,
- "title": article.title,
- "link": article.link,
- "content": article.content,
- "readed": article.readed,
- "like": article.like,
- "date": article.date,
- "retrieved_date": article.retrieved_date,
- "feed_id": article.source.id,
- "feed_name": article.source.title
- }
- for article in result]
- )
-
- def put(self, id):
- """
- Update an article.
- It is only possible to update the status ('like' and 'readed') of an article.
- """
- args = self.reqparse.parse_args()
- article = Article.query.filter(Article.id == id).first()
- if article is not None and article.source.subscriber.id == g.user.id:
- if None is not args.get('like', None):
- article.like = args['like']
- if None is not args.get('readed', None):
- article.readed = args['readed']
- db.session.commit()
-
- try:
- fastsearch.delete_article(g.user.id, article.feed_id, article.id)
- except:
- pass
-
- return jsonify({"message":"ok"})
- else:
- return jsonify({'message': 'Article not found.'})
-
- def delete(self, id):
- """
- Delete an article.
- """
- article = Article.query.filter(Article.id == id).first()
- if article is not None and article.source.subscriber.id == g.user.id:
- db.session.delete(article)
- db.session.commit()
- return jsonify({"message":"ok"})
- else:
- return jsonify({'message': 'Article not found.'})
-
-api.add_resource(ArticleListAPI, '/api/v1.0/articles', endpoint = 'articles.json')
-api.add_resource(ArticleAPI, '/api/v1.0/articles/<int:id>', endpoint = 'article.json')
-
-class FeedListAPI(Resource):
- """
- Defines a RESTful API for Feed elements.
- """
- method_decorators = [authenticate]
-
- def __init__(self):
- self.reqparse = reqparse.RequestParser()
- self.reqparse.add_argument('title', type = unicode, default = "", location = 'json')
- self.reqparse.add_argument('description', type = unicode, default = "", location = 'json')
- self.reqparse.add_argument('link', type = unicode, location = 'json')
- self.reqparse.add_argument('site_link', type = unicode, default = "", location = 'json')
- self.reqparse.add_argument('enabled', type = bool, default = True ,location = 'json')
- super(FeedListAPI, self).__init__()
-
- def get(self):
- """
- Returns a list of feeds.
- """
- return jsonify(result= [{
- "id": feed.id,
- "title": feed.title,
- "description": feed.description,
- "link": feed.link,
- "site_link": feed.site_link,
- "enabled": feed.enabled,
- "created_date": feed.created_date
- }
- for feed in g.user.feeds]
- )
-
- def post(self):
- """
- POST method - Create a new feed.
- """
- args = self.reqparse.parse_args()
- feed_dict = {}
- for k, v in args.iteritems():
- if v != None:
- feed_dict[k] = v
- else:
- return jsonify({'message': 'missing argument: %s' % (k,)})
- new_feed = Feed(title=feed_dict["title"], description=feed_dict["description"],
- link=feed_dict["link"], site_link=feed_dict["site_link"],
- enabled=feed_dict["enabled"])
- g.user.feeds.append(new_feed)
- try:
- db.session.commit()
- return jsonify({"message":"ok"})
- except:
- return jsonify({'message': 'Impossible to create the feed.'})
-
-class FeedAPI(Resource):
- """
- Defines a RESTful API for Feed elements.
- """
- method_decorators = [authenticate]
-
- def __init__(self):
- self.reqparse = reqparse.RequestParser()
- self.reqparse.add_argument('title', type = unicode, location = 'json')
- self.reqparse.add_argument('description', type = unicode, location = 'json')
- self.reqparse.add_argument('link', type = unicode, location = 'json')
- self.reqparse.add_argument('site_link', type = unicode, location = 'json')
- self.reqparse.add_argument('enabled', type = bool ,location = 'json')
- super(FeedAPI, self).__init__()
-
- def get(self, id=None):
- """
- Returns a feed.
- """
- result = []
- if id is not None:
- feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
- if feed is not None:
- result.append(feed)
- return jsonify(result= [{
- "id": feed.id,
- "title": feed.title,
- "description": feed.description,
- "link": feed.link,
- "site_link": feed.site_link,
- "nb_articles": feed.articles.count()
- }
- for feed in result]
- )
- return jsonify({'message': 'Feed not found'})
-
- def put(self, id):
- """
- Update a feed.
- """
- args = self.reqparse.parse_args()
- feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
- if feed is not None:
- if None is not args.get('title', None):
- feed.title = args['title']
- if None is not args.get('description', None):
- feed.description = args['description']
- if None is not args.get('link', None):
- feed.link = args['link']
- if None is not args.get('site_link', None):
- feed.site_link = args['site_link']
- if None is not args.get('enabled', None):
- feed.enabled = args['enabled']
- db.session.commit()
- return jsonify({"message":"ok"})
- else:
- return jsonify({'message': 'Feed not found.'})
-
- def delete(self, id):
- """
- Delete a feed.
- """
- feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
- if feed is not None:
- db.session.delete(feed)
- db.session.commit()
- return jsonify({"message":"ok"})
- else:
- return jsonify({'message': 'Feed not found.'})
-
-api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json')
-api.add_resource(FeedAPI, '/api/v1.0/feeds/<int:id>', endpoint = 'feed.json') \ No newline at end of file
diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py
index 89fa0860..a7f780df 100644
--- a/pyaggr3g470r/search.py
+++ b/pyaggr3g470r/search.py
@@ -102,7 +102,7 @@ def delete_article(user_id, feed_id, article_id):
try:
ix = open_dir(indexdir)
except (EmptyIndexError, OSError):
- raise EmptyIndexError
+ return
writer = ix.writer()
document = And([Term("user_id", user_id), Term("feed_id", feed_id),
Term("article_id", article_id)])
diff --git a/pyaggr3g470r/static/js/articles.js b/pyaggr3g470r/static/js/articles.js
index 51273f3d..312a5cb6 100644
--- a/pyaggr3g470r/static/js/articles.js
+++ b/pyaggr3g470r/static/js/articles.js
@@ -18,6 +18,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+API_ROOT = 'api/v2.0/'
+
if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
+function ($) {
@@ -78,7 +80,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// Encode your data as JSON.
data: data,
// This is the type of data you're expecting back from the server.
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
@@ -114,7 +116,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// Encode your data as JSON.
data: data,
// This is the type of data you're expecting back from the server.
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
@@ -132,7 +134,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
// sends the updates to the server
$.ajax({
type: 'DELETE',
- url: "/api/v1.0/articles/"+article_id,
+ url: API_ROOT + "article/" + article_id,
success: function (result) {
//console.log(result);
},
diff --git a/pyaggr3g470r/templates/edit_feed.html b/pyaggr3g470r/templates/edit_feed.html
index 1238e257..a6b28ded 100644
--- a/pyaggr3g470r/templates/edit_feed.html
+++ b/pyaggr3g470r/templates/edit_feed.html
@@ -17,7 +17,6 @@
{{ form.enabled.label }}
{{ form.enabled(class_="checkbox") }}
-
<br />
{{ form.submit(class_="btn btn-default") }}
</form>
diff --git a/pyaggr3g470r/templates/feed.html b/pyaggr3g470r/templates/feed.html
index 21db7ebe..9910ccf7 100644
--- a/pyaggr3g470r/templates/feed.html
+++ b/pyaggr3g470r/templates/feed.html
@@ -14,6 +14,16 @@
({{ ((feed.articles.all()|count * 100 ) / nb_articles) | round(2, 'floor') }}% {{ _('of the database') }})
{% endif %}
.<br />
+ {% if feed.error_count > 2 %}
+ <b>{{ _("That feed has encountered too much consecutive errors and won't be retreived anymore") }}</b>
+ {% elif feed.error_count > 0 %}
+ {{ _("That feed has encountered some errors but that counter will be reinitialized at the next successful retreiving") }}
+ {% endif %}
+ .<br />
+ {% if feed.last_error %}
+ {{ _("Here's the last error encountered while retreiving this feed:") }} <pre>{{ feed.last_error }}</pre>
+ {% endif %}
+
{{ _('Address of the feed') }}: <a href="{{ feed.link }}">{{ feed.link }}</a><br />
{% if feed.site_link != "" %}
{{ _('Address of the site') }}: <a href="{{ feed.site_link }}">{{ feed.site_link }}</a>
diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html
index a00f962e..8170a99d 100644
--- a/pyaggr3g470r/templates/home.html
+++ b/pyaggr3g470r/templates/home.html
@@ -21,6 +21,9 @@
{% for fid, nbunread in unread|dictsort(by='value')|reverse %}
<li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
{% if feed_id == fid %}<b>{% endif %}
+ {% if in_error.get(fid, 0) > 0 %}
+ <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span>
+ {% endif %}
<span id="unread-{{ fid }}" class="badge pull-right">{{ nbunread }}</span>
{{ feeds[fid]|safe }}
{% if feed_id == fid %}</b>{% endif %}
@@ -36,6 +39,9 @@
{% endfor %}
{% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %}
<li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
+ {% if in_error.get(fid, 0) > 0 %}
+ <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span>
+ {% endif %}
{% if feed_id == fid %}<b>{% endif %}
{{ ftitle|safe }}
{% if feed_id == fid %}</b>{% endif %}
diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html
index 4dc62350..6b929bf3 100644
--- a/pyaggr3g470r/templates/layout.html
+++ b/pyaggr3g470r/templates/layout.html
@@ -7,9 +7,9 @@
<meta name="description" content="pyAggr3g470r is a web-based news aggregator." />
<meta name="author" content="" />
<title>{% if head_title %}{{ head_title }} - {% endif %}pyAggr3g470r</title>
- <link rel="shortcut icon" href="{{ url_for('.static', filename='img/favicon.png') }}" />
+ <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.png') }}" />
<!-- Bootstrap core CSS -->
- <link href="{{ url_for('.static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
+ <link href="{{ url_for('static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
<!-- Add custom CSS here -->
<style>
body {
@@ -155,9 +155,9 @@
<!-- Bootstrap core JavaScript -->
<!-- Placed at the end of the document so the pages load faster -->
- <script src="{{ url_for('.static', filename = 'js/jquery.js') }}"></script>
- <script src="{{ url_for('.static', filename = 'js/bootstrap.js') }}"></script>
- <script src="{{ url_for('.static', filename = 'js/articles.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/jquery.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/bootstrap.js') }}"></script>
+ <script src="{{ url_for('static', filename = 'js/articles.js') }}"></script>
<script type="text/javascript" class="source">
if (window.location.href.indexOf("filter_=all") > -1){
$("#tab-all").attr('class', "active");
diff --git a/pyaggr3g470r/templates/profile.html b/pyaggr3g470r/templates/profile.html
index 6f22857c..f879b054 100644
--- a/pyaggr3g470r/templates/profile.html
+++ b/pyaggr3g470r/templates/profile.html
@@ -14,7 +14,10 @@
{{ form.password.label }}
{{ form.password(class_="form-control") }} {% for error in form.password.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
-
+ {{ form.password_conf.label }}
+ {{ form.password_conf(class_="form-control") }} {% for error in form.password_conf.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
+ {{ form.refresh_rate.label }}
+ {{ form.refresh_rate(class_="form-control") }} {% for error in form.refresh_rate.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
<br />
{{ form.submit(class_="btn") }}
</form>
diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py
index 972909af..1fc84ff4 100755
--- a/pyaggr3g470r/utils.py
+++ b/pyaggr3g470r/utils.py
@@ -51,9 +51,10 @@ from collections import Counter
from contextlib import contextmanager
import conf
-from pyaggr3g470r import db
+from flask import g
from pyaggr3g470r.models import User, Feed, Article
+
# regular expression to check URL
url_finders = [
re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \
@@ -135,7 +136,7 @@ def import_opml(email, opml_content):
return nb
nb = read(subscriptions)
- db.session.commit()
+ g.db.session.commit()
return nb
def import_json(email, json_content):
@@ -158,7 +159,7 @@ def import_json(email, json_content):
enabled=feed["enabled"])
user.feeds.append(new_feed)
nb_feeds += 1
- db.session.commit()
+ g.db.session.commit()
# Create articles
for feed in json_account["result"]:
@@ -178,7 +179,7 @@ def import_json(email, json_content):
user_feed.articles.append(new_article)
nb_articles += 1
- db.session.commit()
+ g.db.session.commit()
return nb_feeds, nb_articles
diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py
new file mode 100644
index 00000000..029dcb7d
--- /dev/null
+++ b/pyaggr3g470r/views/__init__.py
@@ -0,0 +1,5 @@
+from .views import *
+from .api import *
+
+from .article import article_bp, articles_bp
+from .feed import feed_bp, feeds_bp
diff --git a/pyaggr3g470r/views/api/__init__.py b/pyaggr3g470r/views/api/__init__.py
new file mode 100644
index 00000000..e11cdd95
--- /dev/null
+++ b/pyaggr3g470r/views/api/__init__.py
@@ -0,0 +1,31 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.2 $"
+__date__ = "$Date: 2014/06/18 $"
+__revision__ = "$Date: 2014/07/05 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "AGPLv3"
+
+from pyaggr3g470r.views.api import article, feed
+
+__all__ = ['article', 'feed']
diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py
new file mode 100644
index 00000000..17881412
--- /dev/null
+++ b/pyaggr3g470r/views/api/article.py
@@ -0,0 +1,58 @@
+from flask import g
+import dateutil.parser
+
+from pyaggr3g470r.controllers import ArticleController
+from pyaggr3g470r.views.api.common import PyAggAbstractResource,\
+ PyAggResourceNew, \
+ PyAggResourceExisting, \
+ PyAggResourceMulti
+
+
+ARTICLE_ATTRS = {'feed_id': {'type': str},
+ 'entry_id': {'type': str},
+ 'link': {'type': str},
+ 'title': {'type': str},
+ 'readed': {'type': bool}, 'like': {'type': bool},
+ 'content': {'type': str},
+ 'date': {'type': str}, 'retrieved_date': {'type': str}}
+
+
+class ArticleNewAPI(PyAggResourceNew):
+ controller_cls = ArticleController
+ attrs = ARTICLE_ATTRS
+ to_date = ['date', 'retrieved_date']
+
+
+class ArticleAPI(PyAggResourceExisting):
+ controller_cls = ArticleController
+ attrs = ARTICLE_ATTRS
+ to_date = ['date', 'retrieved_date']
+
+
+class ArticlesAPI(PyAggResourceMulti):
+ controller_cls = ArticleController
+ attrs = ARTICLE_ATTRS
+ to_date = ['date', 'retrieved_date']
+
+
+class ArticlesChallenge(PyAggAbstractResource):
+ controller_cls = ArticleController
+ attrs = {'ids': {'type': list, 'default': []}}
+ to_date = ['date', 'retrieved_date']
+
+ def get(self):
+ parsed_args = self.reqparse_args()
+ for id_dict in parsed_args['ids']:
+ for key in self.to_date:
+ if key in id_dict:
+ id_dict[key] = dateutil.parser.parse(id_dict[key])
+
+ return self.controller.challenge(parsed_args['ids'])
+
+
+g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
+g.api.add_resource(ArticleAPI, '/article/<int:obj_id>',
+ endpoint='article.json')
+g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json')
+g.api.add_resource(ArticlesChallenge, '/articles/challenge',
+ endpoint='articles_challenge.json')
diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py
new file mode 100644
index 00000000..4f703712
--- /dev/null
+++ b/pyaggr3g470r/views/api/common.py
@@ -0,0 +1,217 @@
+"""For a given resources, classes in the module intend to create the following
+routes :
+ GET resource/<id>
+ -> to retreive one
+ POST resource
+ -> to create one
+ PUT resource/<id>
+ -> to update one
+ DELETE resource/<id>
+ -> to delete one
+
+ GET resources
+ -> to retreive several
+ POST resources
+ -> to create several
+ PUT resources
+ -> to update several
+ DELETE resources
+ -> to delete several
+"""
+import json
+import logging
+import dateutil.parser
+from copy import deepcopy
+from functools import wraps
+from werkzeug.exceptions import Unauthorized, BadRequest
+from flask import request, g, session, Response
+from flask.ext.restful import Resource, reqparse
+
+from pyaggr3g470r.lib.utils import default_handler
+from pyaggr3g470r.models import User
+
+logger = logging.getLogger(__name__)
+
+
+def authenticate(func):
+ """
+ Decorator for the authentication to the web services.
+ """
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ logged_in = False
+ if not getattr(func, 'authenticated', True):
+ logged_in = True
+ # authentication based on the session (already logged on the site)
+ elif 'email' in session or g.user.is_authenticated():
+ logged_in = True
+ else:
+ # authentication via HTTP only
+ auth = request.authorization
+ user = User.query.filter(User.nickname == auth.username).first()
+ if user and user.check_password(auth.password) \
+ and user.activation_key == "":
+ g.user = user
+ logged_in = True
+
+ if logged_in:
+ return func(*args, **kwargs)
+ raise Unauthorized({'WWWAuthenticate': 'Basic realm="Login Required"'})
+ return wrapper
+
+
+def to_response(func):
+ """Will cast results of func as a result, and try to extract
+ a status_code for the Response object"""
+ def wrapper(*args, **kwargs):
+ status_code = 200
+ result = func(*args, **kwargs)
+ if isinstance(result, Response):
+ return result
+ elif isinstance(result, tuple):
+ result, status_code = result
+ return Response(json.dumps(result, default=default_handler),
+ status=status_code)
+ return wrapper
+
+
+class PyAggAbstractResource(Resource):
+ method_decorators = [authenticate, to_response]
+ attrs = {}
+ to_date = [] # list of fields to cast to datetime
+
+ def __init__(self, *args, **kwargs):
+ super(PyAggAbstractResource, self).__init__(*args, **kwargs)
+
+ @property
+ def controller(self):
+ return self.controller_cls(getattr(g.user, 'id', None))
+
+ def reqparse_args(self, req=None, strict=False, default=True, args=None):
+ """
+ strict: bool
+ if True will throw 400 error if args are defined and not in request
+ default: bool
+ if True, won't return defaults
+ args: dict
+ the args to parse, if None, self.attrs will be used
+ """
+ parser = reqparse.RequestParser()
+ for attr_name, attrs in (args or self.attrs).items():
+ if attrs.pop('force_default', False):
+ parser.add_argument(attr_name, location='json', **attrs)
+ elif not default and (not request.json
+ or request.json and attr_name not in request.json):
+ continue
+ else:
+ parser.add_argument(attr_name, location='json', **attrs)
+ parsed = parser.parse_args(strict=strict) if req is None \
+ else parser.parse_args(req, strict=strict)
+ for field in self.to_date:
+ if parsed.get(field):
+ try:
+ parsed[field] = dateutil.parser.parse(parsed[field])
+ except Exception:
+ logger.exception('failed to parse %r', parsed[field])
+ return parsed
+
+
+class PyAggResourceNew(PyAggAbstractResource):
+
+ def post(self):
+ """Create a single new object"""
+ return self.controller.create(**self.reqparse_args()), 201
+
+
+class PyAggResourceExisting(PyAggAbstractResource):
+
+ def get(self, obj_id=None):
+ """Retreive a single object"""
+ return self.controller.get(id=obj_id)
+
+ def put(self, obj_id=None):
+ """update an object, new attrs should be passed in the payload"""
+ args = self.reqparse_args(default=False)
+ new_values = {key: args[key] for key in
+ set(args).intersection(self.attrs)}
+ self.controller.update({'id': obj_id}, new_values)
+
+ def delete(self, obj_id=None):
+ """delete a object"""
+ self.controller.delete(obj_id)
+ return None, 204
+
+
+class PyAggResourceMulti(PyAggAbstractResource):
+
+ def get(self):
+ """retreive several objects. filters can be set in the payload on the
+ different fields of the object, and a limit can be set in there as well
+ """
+ if 'application/json' != request.headers.get('Content-Type'):
+ raise BadRequest("Content-Type must be application/json")
+ limit = request.json.pop('limit', 10)
+ if not limit:
+ return [res for res in self.controller.read(**request.json).all()]
+ return [res for res in self.controller.read(**request.json).limit(limit)]
+
+ def post(self):
+ """creating several objects. payload should be a list of dict.
+ """
+ if 'application/json' != request.headers.get('Content-Type'):
+ raise BadRequest("Content-Type must be application/json")
+ status = 201
+ results = []
+ for attrs in request.json:
+ try:
+ results.append(self.controller.create(**attrs).id)
+ except Exception as error:
+ status = 206
+ results.append(str(error))
+ # if no operation succeded, it's not partial anymore, returning err 500
+ if status == 206 and results.count('ok') == 0:
+ status = 500
+ return results, status
+
+ def put(self):
+ """creating several objects. payload should be:
+ >>> payload
+ [[obj_id1, {attr1: val1, attr2: val2}]
+ [obj_id2, {attr1: val1, attr2: val2}]]
+ """
+ if 'application/json' != request.headers.get('Content-Type'):
+ raise BadRequest("Content-Type must be application/json")
+ status = 200
+ results = []
+ for obj_id, attrs in request.json:
+ try:
+ new_values = {key: attrs[key] for key in
+ set(attrs).intersection(self.attrs)}
+ self.controller.update({'id': obj_id}, new_values)
+ results.append('ok')
+ except Exception as error:
+ status = 206
+ results.append(str(error))
+ # if no operation succeded, it's not partial anymore, returning err 500
+ if status == 206 and results.count('ok') == 0:
+ status = 500
+ return results, status
+
+ def delete(self):
+ """will delete several objects,
+ a list of their ids should be in the payload"""
+ if 'application/json' != request.headers.get('Content-Type'):
+ raise BadRequest("Content-Type must be application/json")
+ status = 204
+ results = []
+ for obj_id in request.json:
+ try:
+ self.controller.delete(obj_id)
+ results.append('ok')
+ except Exception as error:
+ status = 206
+ results.append(error)
+ # if no operation succeded, it's not partial anymore, returning err 500
+ if status == 206 and results.count('ok') == 0:
+ status = 500
+ return results, status
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
new file mode 100644
index 00000000..898e30b0
--- /dev/null
+++ b/pyaggr3g470r/views/api/feed.py
@@ -0,0 +1,57 @@
+from flask import g
+
+from pyaggr3g470r.controllers.feed import FeedController, \
+ DEFAULT_MAX_ERROR, DEFAULT_LIMIT
+
+from pyaggr3g470r.views.api.common import PyAggAbstractResource, \
+ PyAggResourceNew, \
+ PyAggResourceExisting, \
+ PyAggResourceMulti
+
+
+FEED_ATTRS = {'title': {'type': str},
+ 'description': {'type': str},
+ 'link': {'type': str},
+ 'site_link': {'type': str},
+ 'enabled': {'type': bool, 'default': True},
+ 'etag': {'type': str, 'default': ''},
+ 'last_modified': {'type': str},
+ 'last_retreived': {'type': str},
+ 'last_error': {'type': str},
+ 'error_count': {'type': int, 'default': 0}}
+
+
+class FeedNewAPI(PyAggResourceNew):
+ controller_cls = FeedController
+ attrs = FEED_ATTRS
+ to_date = ['date', 'last_retreived']
+
+
+class FeedAPI(PyAggResourceExisting):
+ controller_cls = FeedController
+ attrs = FEED_ATTRS
+ to_date = ['date', 'last_retreived']
+
+
+class FeedsAPI(PyAggResourceMulti):
+ controller_cls = FeedController
+ attrs = FEED_ATTRS
+ to_date = ['date', 'last_retreived']
+
+
+class FetchableFeedAPI(PyAggAbstractResource):
+ controller_cls = FeedController
+ to_date = ['date', 'last_retreived']
+ attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR},
+ 'limit': {'type': int, 'default': DEFAULT_LIMIT}}
+
+ def get(self):
+ return [feed for feed in self.controller.list_fetchable(
+ **self.reqparse_args())]
+
+
+g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')
+g.api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json')
+g.api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json')
+g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable',
+ endpoint='fetchable_feed.json')
diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py
new file mode 100644
index 00000000..66cc0f37
--- /dev/null
+++ b/pyaggr3g470r/views/article.py
@@ -0,0 +1,53 @@
+from flask import Blueprint, g, render_template, redirect
+from sqlalchemy import desc
+
+from pyaggr3g470r import controllers, utils
+from pyaggr3g470r.decorators import pyagg_default_decorator
+
+articles_bp = Blueprint('articles', __name__, url_prefix='/articles')
+article_bp = Blueprint('article', __name__, url_prefix='/article')
+
+
+@articles_bp.route('/<feed_id>', methods=['GET'])
+@articles_bp.route('/<feed_id>/<int:nb_articles>', methods=['GET'])
+@pyagg_default_decorator
+def articles(feed_id=None, nb_articles=-1):
+ """List articles of a feed. The administrator of the platform is able to
+ access to this view for every users."""
+ feed = controllers.FeedController(g.user.id).get(id=feed_id)
+ feed.articles = controllers.ArticleController(g.user.id)\
+ .read(feed_id=feed.id)\
+ .order_by(desc("Article.date"))
+ if len(feed.articles.all()) <= nb_articles:
+ nb_articles = -1
+ if nb_articles == -1:
+ feed.articles = feed.articles.limit(nb_articles)
+ return render_template('articles.html', feed=feed, nb_articles=nb_articles)
+
+
+@article_bp.route('/redirect/<int:article_id>', methods=['GET'])
+@pyagg_default_decorator
+def redirect_to_article(article_id):
+ article = controllers.ArticleController(g.user.id).get(id=article_id)
+ return redirect(article.link)
+
+
+@article_bp.route('/<int:article_id>', methods=['GET'])
+@pyagg_default_decorator
+def article(article_id=None):
+ """
+ Presents the content of an article.
+ """
+ article = controllers.ArticleController(g.user.id).get(id=article_id)
+ previous_article = article.previous_article()
+ if previous_article is None:
+ previous_article = article.source.articles[0]
+ next_article = article.next_article()
+ if next_article is None:
+ next_article = article.source.articles[-1]
+
+ return render_template('article.html',
+ head_title=utils.clear_string(article.title),
+ article=article,
+ previous_article=previous_article,
+ next_article=next_article)
diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py
new file mode 100644
index 00000000..2af502a7
--- /dev/null
+++ b/pyaggr3g470r/views/feed.py
@@ -0,0 +1,50 @@
+from datetime import datetime
+from flask import Blueprint, g, render_template
+
+from pyaggr3g470r import controllers, utils
+from pyaggr3g470r.decorators import pyagg_default_decorator, \
+ feed_access_required
+
+feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
+feed_bp = Blueprint('feed', __name__, url_prefix='/feed')
+
+@feeds_bp.route('/', methods=['GET'])
+def feeds():
+ "Lists the subscribed feeds in a table."
+ return render_template('feeds.html',
+ feeds=controllers.FeedController(g.user.id).read())
+
+
+@feed_bp.route('/<int:feed_id>', methods=['GET'])
+@pyagg_default_decorator
+@feed_access_required
+def feed(feed_id=None):
+ "Presents detailed information about a feed."
+ feed = controllers.FeedController(g.user.id).get(id=feed_id)
+ word_size = 6
+ articles = controllers.ArticleController(g.user.id)\
+ .read(feed_id=feed_id).all()
+ nb_articles = controllers.ArticleController(g.user.id).read().count()
+ top_words = utils.top_words(articles, n=50, size=int(word_size))
+ tag_cloud = utils.tag_cloud(top_words)
+
+ today = datetime.now()
+ try:
+ last_article = articles[0].date
+ first_article = articles[-1].date
+ delta = last_article - first_article
+ average = round(float(len(articles)) / abs(delta.days), 2)
+ except:
+ last_article = datetime.fromtimestamp(0)
+ first_article = datetime.fromtimestamp(0)
+ delta = last_article - first_article
+ average = 0
+ elapsed = today - last_article
+
+ return render_template('feed.html',
+ head_title=utils.clear_string(feed.title),
+ feed=feed, tag_cloud=tag_cloud,
+ first_post_date=first_article,
+ end_post_date=last_article,
+ nb_articles=nb_articles,
+ average=average, delta=delta, elapsed=elapsed)
diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views/views.py
index e5e07cde..5279d7fe 100644
--- a/pyaggr3g470r/views.py
+++ b/pyaggr3g470r/views/views.py
@@ -27,9 +27,14 @@ __copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "AGPLv3"
import os
+import json
+import string
+import random
+import hashlib
import datetime
from collections import namedtuple
-from flask import abort, render_template, request, flash, session, \
+from bootstrap import application as app, db
+from flask import render_template, request, flash, session, Response, \
url_for, redirect, g, current_app, make_response, jsonify
from flask.ext.login import LoginManager, login_user, logout_user, \
login_required, current_user, AnonymousUserMixin
@@ -37,17 +42,17 @@ from flask.ext.principal import Principal, Identity, AnonymousIdentity, \
identity_changed, identity_loaded, Permission,\
RoleNeed, UserNeed
from flask.ext.babel import gettext
-from sqlalchemy import desc, func, or_
+from sqlalchemy import func, or_
from sqlalchemy.exc import IntegrityError
from werkzeug import generate_password_hash
import conf
from pyaggr3g470r import utils, notifications, export, duplicate
-from pyaggr3g470r import app, db, allowed_file, babel
from pyaggr3g470r.models import User, Feed, Article, Role
from pyaggr3g470r.decorators import feed_access_required
from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \
ProfileForm, InformationMessageForm, RecoverPasswordForm
+from pyaggr3g470r.controllers import FeedController
if not conf.ON_HEROKU:
import pyaggr3g470r.search as fastsearch
@@ -118,7 +123,7 @@ def redirect_url(default='home'):
request.referrer or \
url_for(default)
-@babel.localeselector
+@g.babel.localeselector
def get_locale():
"""
Called before each request to give us a chance to choose
@@ -126,7 +131,7 @@ def get_locale():
"""
return request.accept_languages.best_match(conf.LANGUAGES.keys())
-@babel.timezoneselector
+@g.babel.timezoneselector
def get_timezone():
try:
return conf.TIME_ZONE[get_locale()]
@@ -152,11 +157,13 @@ def login():
login_user(user)
g.user = user
session['email'] = form.email.data
- identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+ identity_changed.send(current_app._get_current_object(),
+ identity=Identity(user.id))
flash(gettext("Logged in successfully."), 'success')
return redirect(url_for('home'))
return render_template('login.html', form=form)
+
@app.route('/logout')
@login_required
def logout():
@@ -240,23 +247,14 @@ def home():
unread = db.session.query(Article.feed_id, func.count(Article.id))\
.filter(Article.readed == False, Article.user_id == g.user.id)\
.group_by(Article.feed_id).all()
+ in_error = {feed.id: feed.error_count for feed in
+ FeedController(g.user.id).read(error_count__gt=0).all()}
def gen_url(filter_=filter_, limit=limit, feed=feed_id):
return '?filter_=%s&limit=%s&feed=%d' % (filter_, limit, feed)
return render_template('home.html', gen_url=gen_url, feed_id=feed_id,
filter_=filter_, limit=limit, feeds=feeds,
- unread=dict(unread), articles=articles.all())
-
-
-@app.route('/article/redirect/<int:article_id>', methods=['GET'])
-@login_required
-def redirect_to_article(article_id):
- article = Article.query.filter(Article.id == article_id,
- Article.user_id == g.user.id).first()
- if article is None:
- abort(404)
- article.readed = True
- db.session.commit()
- return redirect(article.link)
+ unread=dict(unread), articles=articles.all(),
+ in_error=in_error)
@app.route('/fetch', methods=['GET'])
@@ -278,71 +276,6 @@ def about():
"""
return render_template('about.html')
-@app.route('/feeds', methods=['GET'])
-@login_required
-def feeds():
- """
- Lists the subscribed feeds in a table.
- """
- user = User.query.filter(User.email == g.user.email).first()
- return render_template('feeds.html', feeds=user.feeds)
-
-@app.route('/feed/<int:feed_id>', methods=['GET'])
-@login_required
-@feed_access_required
-def feed(feed_id=None):
- """
- Presents detailed information about a feed.
- """
- feed = Feed.query.filter(Feed.id == feed_id).first()
- word_size = 6
- articles = feed.articles.all()
- nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all())
- top_words = utils.top_words(articles, n=50, size=int(word_size))
- tag_cloud = utils.tag_cloud(top_words)
-
- today = datetime.datetime.now()
- try:
- last_article = articles[0].date
- first_article = articles[-1].date
- delta = last_article - first_article
- average = round(float(len(articles)) / abs(delta.days), 2)
- except:
- last_article = datetime.datetime.fromtimestamp(0)
- first_article = datetime.datetime.fromtimestamp(0)
- delta = last_article - first_article
- average = 0
- elapsed = today - last_article
-
- return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \
- first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \
- average=average, delta=delta, elapsed=elapsed)
-
-@app.route('/article/<int:article_id>', methods=['GET'])
-@login_required
-def article(article_id=None):
- """
- Presents the content of an article.
- """
- article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first()
- if article is not None:
- if not article.readed:
- article.readed = True
- db.session.commit()
-
- previous_article = article.previous_article()
- if previous_article is None:
- previous_article = article.source.articles[0]
- next_article = article.next_article()
- if next_article is None:
- next_article = article.source.articles[-1]
-
- return render_template('article.html', head_title=utils.clear_string(article.title),
- article=article,
- previous_article=previous_article, next_article=next_article)
- flash(gettext("This article do not exist."), 'warning')
- return redirect(url_for('home'))
-
@app.route('/mark_as/<string:new_value>', methods=['GET'])
@app.route('/mark_as/<string:new_value>/feed/<int:feed_id>', methods=['GET'])
@@ -403,24 +336,6 @@ def delete(article_id=None):
flash(gettext('This article do not exist.'), 'danger')
return redirect(url_for('home'))
-@app.route('/articles/<feed_id>', methods=['GET'])
-@app.route('/articles/<feed_id>/<int:nb_articles>', methods=['GET'])
-@login_required
-@feed_access_required
-def articles(feed_id=None, nb_articles=-1):
- """
- List articles of a feed.
- The administrator of the platform is able to access to this view for every users.
- """
- feed = Feed.query.filter(Feed.id == feed_id).first()
- new_feed = feed
- if len(feed.articles.all()) <= nb_articles:
- nb_articles = -1
- if nb_articles == -1:
- nb_articles = int(1e9)
- new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \
- Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles)
- return render_template('articles.html', feed=new_feed, nb_articles=nb_articles)
@app.route('/favorites', methods=['GET'])
@login_required
@@ -591,7 +506,7 @@ def management():
if None != request.files.get('opmlfile', None):
# Import an OPML file
data = request.files.get('opmlfile', None)
- if not allowed_file(data.filename):
+ if not g.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
@@ -604,7 +519,7 @@ def management():
elif None != request.files.get('jsonfile', None):
# Import an account
data = request.files.get('jsonfile', None)
- if not allowed_file(data.filename):
+ if not g.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
@@ -627,7 +542,7 @@ def management():
@app.route('/history', methods=['GET'])
@login_required
def history():
- user = User.query.filter(User.id == g.user.id).first()
+ #user = User.query.filter(User.id == g.user.id).first()
return render_template('history.html')
@app.route('/bookmarklet', methods=['GET'])
@@ -639,7 +554,7 @@ def edit_feed(feed_id=None):
"""
Add or edit a feed.
"""
- feed = Feed.query.filter(Feed.id == feed_id).first()
+ feed = FeedController(g.user.id).get(id=feed_id)
form = AddFeedForm()
if request.method == 'POST':
@@ -653,7 +568,7 @@ def edit_feed(feed_id=None):
return redirect('/edit_feed/' + str(feed_id))
else:
# Create a new feed
- existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data]
+ existing_feed = [f for f in g.user.feeds if feed.link == form.link.data]
if len(existing_feed) == 0:
new_feed = Feed(title=form.title.data, description="", link=form.link.data, \
site_link=form.site_link.data, enabled=form.enabled.data)
@@ -678,7 +593,7 @@ def edit_feed(feed_id=None):
# Enable the user to add a feed with a bookmarklet
if None is not request.args.get('url', None):
- existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)]
+ existing_feed = [f for f in g.user.feeds if feed.link == request.args.get('url', None)]
if len(existing_feed) == 0:
g.user.feeds.append(Feed(link=request.args.get('url', None)))
db.session.commit()
@@ -717,7 +632,9 @@ def profile():
if form.password.data != "":
user.set_password(form.password.data)
db.session.commit()
- flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success')
+ flash("%s %s %s" % (gettext('User'), user.nickname,
+ gettext('successfully updated.')),
+ 'success')
return redirect(url_for('profile'))
else:
return render_template('profile.html', form=form)
@@ -777,8 +694,6 @@ def recover():
Enables the user to recover its account when he has forgotten
its password.
"""
- import string
- import random
form = RecoverPasswordForm()
if request.method == 'POST':
@@ -920,7 +835,6 @@ def disable_user(user_id=None):
flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger')
else:
- import random, hashlib
user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86]
flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success')
db.session.commit()
bgstack15