Merged in jaesivsm/pyaggr3g470r/evol/api (pull request #6)

Evolution Arch, API and new crawler

36 files changed, 1334 insertions, 722 deletions

diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py
index f3f784f4..e69de29b 100644
--- a/pyaggr3g470r/__init__.py
+++ b/pyaggr3g470r/__init__.py
@@ -1,48 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-from flask import Flask
-from flask.ext.sqlalchemy import SQLAlchemy
-from flask.ext.babel import Babel
-from flask.ext.babel import format_datetime
-
-import conf
-
-# Create Flask application
-app = Flask(__name__)
-app.debug = conf.WEBSERVER_DEBUG
-
-# Create dummy secrey key so we can use sessions
-app.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None)
-if not app.config['SECRET_KEY']:
-    app.config['SECRET_KEY'] = os.urandom(12)
-app.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI
-db = SQLAlchemy(app)
-
-app.config['RECAPTCHA_USE_SSL'] = True
-app.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY
-app.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY
-
-if conf.ON_HEROKU:
-    from flask_sslify import SSLify
-    sslify = SSLify(app)
-
-ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json'])
-
-def allowed_file(filename):
-    """
-    Check if the uploaded file is allowed.
-    """
-    return '.' in filename and \
-            filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
-
-babel = Babel(app)
-
-app.jinja_env.filters['datetime'] = format_datetime
-
-# Views
-from flask.ext.restful import Api
-api = Api(app)
-
-from pyaggr3g470r import views, rest
diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py
new file mode 100644
index 00000000..d8d1a104
--- /dev/null
+++ b/pyaggr3g470r/controllers/__init__.py
@@ -0,0 +1,6 @@
+from .feed import FeedController
+from .article import ArticleController
+from .user import UserController
+
+
+__all__ = ['FeedController', 'ArticleController', 'UserController']
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py
new file mode 100644
index 00000000..a99e67f3
--- /dev/null
+++ b/pyaggr3g470r/controllers/abstract.py
@@ -0,0 +1,69 @@
+import logging
+from bootstrap import db
+from sqlalchemy import update
+from werkzeug.exceptions import Forbidden, NotFound
+
+logger = logging.getLogger(__name__)
+
+
+class AbstractController(object):
+    _db_cls = None  # reference to the database class
+    _user_id_key = 'user_id'
+
+    def __init__(self, user_id):
+        self.user_id = user_id
+
+    def _to_filters(self, **filters):
+        if self.user_id:
+            filters[self._user_id_key] = self.user_id
+        db_filters = set()
+        for key, value in filters.items():
+            if key.endswith('__gt'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) > value)
+            elif key.endswith('__lt'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) < value)
+            elif key.endswith('__ge'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) >= value)
+            elif key.endswith('__le'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) <= value)
+            elif key.endswith('__ne'):
+                db_filters.add(getattr(self._db_cls, key[:-4]) != value)
+            elif key.endswith('__in'):
+                db_filters.add(getattr(self._db_cls, key[:-4]).in_(value))
+            else:
+                db_filters.add(getattr(self._db_cls, key) == value)
+        return db_filters
+
+    def _get(self, **filters):
+        return self._db_cls.query.filter(*self._to_filters(**filters))
+
+    def get(self, **filters):
+        obj = self._get(**filters).first()
+        if not obj:
+            raise NotFound({'message': 'No %r (%r)'
+                                % (self._db_cls.__class__.__name__, filters)})
+        if getattr(obj, self._user_id_key) != self.user_id:
+            raise Forbidden({'message': 'No authorized to access %r (%r)'
+                                % (self._db_cls.__class__.__name__, filters)})
+        return obj
+
+    def create(self, **attrs):
+        attrs[self._user_id_key] = self.user_id
+        obj = self._db_cls(**attrs)
+        db.session.add(obj)
+        db.session.commit()
+        return obj
+
+    def read(self, **filters):
+        return self._get(**filters)
+
+    def update(self, filters, attrs):
+        result = self._get(**filters).update(attrs, synchronize_session=False)
+        db.session.commit()
+        return result
+
+    def delete(self, obj_id):
+        obj = self.get(id=obj_id)
+        db.session.delete(obj)
+        db.session.commit()
+        return obj
diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py
new file mode 100644
index 00000000..46ca0988
--- /dev/null
+++ b/pyaggr3g470r/controllers/article.py
@@ -0,0 +1,27 @@
+import conf
+from .abstract import AbstractController
+from pyaggr3g470r.models import Article
+
+
+class ArticleController(AbstractController):
+    _db_cls = Article
+
+    def get(self, **filters):
+        article = super(ArticleController, self).get(**filters)
+        if not article.readed:
+            self.update({'id': article.id}, {'readed': True})
+        return article
+
+    def delete(self, obj_id):
+        obj = super(ArticleController, self).delete(obj_id)
+        if not conf.ON_HEROKU:
+            import pyaggr3g470r.search as fastsearch
+            fastsearch.delete_article(self.user_id, obj.feed_id, obj_id)
+        return obj
+
+    def challenge(self, ids):
+        """Will return each id that wasn't found in the database."""
+        for id_ in ids:
+            if self.read(**id_).first():
+                continue
+            yield id_
diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py
new file mode 100644
index 00000000..a2455e2b
--- /dev/null
+++ b/pyaggr3g470r/controllers/feed.py
@@ -0,0 +1,24 @@
+from datetime import datetime, timedelta
+from .abstract import AbstractController
+from pyaggr3g470r.models import Feed
+
+DEFAULT_MAX_ERROR = 3
+DEFAULT_LIMIT = 5
+
+
+class FeedController(AbstractController):
+    _db_cls = Feed
+
+    def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT):
+        from pyaggr3g470r.controllers import UserController
+        now = datetime.now()
+        user = UserController(self.user_id).get(id=self.user_id)
+        max_last = now - timedelta(minutes=user.refresh_rate or 60)
+        feeds = [feed for feed in self.read(user_id=self.user_id,
+                            error_count__lt=max_error, enabled=True,
+                            last_retreived__lt=max_last).limit(limit)]
+
+        if feeds:
+            self.update({'id__in': [feed.id for feed in feeds]},
+                        {'last_retreived': now})
+        return feeds
diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py
new file mode 100644
index 00000000..c6c1d545
--- /dev/null
+++ b/pyaggr3g470r/controllers/user.py
@@ -0,0 +1,7 @@
+from .abstract import AbstractController
+from pyaggr3g470r.models import User
+
+
+class UserController(AbstractController):
+    _db_cls = User
+    _user_id_key = 'id'
diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py
index 3e808793..9e8f9c0b 100644
--- a/pyaggr3g470r/decorators.py
+++ b/pyaggr3g470r/decorators.py
@@ -1,10 +1,12 @@
 #! /usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 from threading import Thread
 from functools import wraps
 
 from flask import g, redirect, url_for, flash
+from flask.ext.babel import gettext
+from flask.ext.login import login_required
 
 from pyaggr3g470r.models import Feed
 
@@ -20,6 +22,7 @@ def async(f):
         thr.start()
     return wrapper
 
+
 def feed_access_required(func):
     """
     This decorator enables to check if a user has access to a feed.
@@ -37,3 +40,11 @@ def feed_access_required(func):
         else:
             return func(*args, **kwargs)
     return decorated
+
+
+def pyagg_default_decorator(func):
+    @login_required
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+    return wrapper
diff --git a/pyaggr3g470r/emails.py b/pyaggr3g470r/emails.py
index 50f94761..9a129718 100644
--- a/pyaggr3g470r/emails.py
+++ b/pyaggr3g470r/emails.py
@@ -31,6 +31,7 @@ from pyaggr3g470r.decorators import async
 
 logger = logging.getLogger(__name__)
 
+
 @async
 def send_async_email(mfrom, mto, msg):
     try:
diff --git a/pyaggr3g470r/forms.py b/pyaggr3g470r/forms.py
index 58abb864..e385c4e2 100644
--- a/pyaggr3g470r/forms.py
+++ b/pyaggr3g470r/forms.py
@@ -29,47 +29,52 @@ __license__ = "GPLv3"
 from flask import flash
 from flask.ext.wtf import Form
 from flask.ext.babel import lazy_gettext
-from wtforms import TextField, TextAreaField, PasswordField, BooleanField, SubmitField, validators
+from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \
+        SubmitField, IntegerField, validators
 from flask.ext.wtf.html5 import EmailField
 from flask_wtf import RecaptchaField
 
 from pyaggr3g470r.models import User
 
 class SignupForm(Form):
-    nickname = TextField(lazy_gettext("Nickname"), [validators.Required(lazy_gettext("Please enter your nickname."))])
-    email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
-    password = PasswordField(lazy_gettext("Password"), [validators.Required(lazy_gettext("Please enter a password.")), validators.Length(min=6, max=100)])
+    nickname = TextField(lazy_gettext("Nickname"),
+            [validators.Required(lazy_gettext("Please enter your nickname."))])
+    email = EmailField(lazy_gettext("Email"),
+            [validators.Length(min=6, max=35),
+             validators.Required(
+                 lazy_gettext("Please enter your email address."))])
+    password = PasswordField(lazy_gettext("Password"),
+            [validators.Required(lazy_gettext("Please enter a password.")),
+             validators.Length(min=6, max=100)])
     recaptcha = RecaptchaField()
     submit = SubmitField(lazy_gettext("Sign up"))
 
-    def __init__(self, *args, **kwargs):
-        Form.__init__(self, *args, **kwargs)
-
     def validate(self):
-        if not Form.validate(self):
-            return False
+        validated = super(SignupForm, self).validate()
         if self.nickname.data != User.make_valid_nickname(self.nickname.data):
             self.nickname.errors.append(lazy_gettext('This nickname has invalid characters. Please use letters, numbers, dots and underscores only.'))
-            return False
-        return True
+            validated = False
+        return validated
+
 
 class SigninForm(Form):
     """
     Sign in form.
     """
-    email = EmailField("Email", [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
-    password = PasswordField(lazy_gettext('Password'), [validators.Required(lazy_gettext("Please enter a password.")), validators.Length(min=6, max=100)])
+    email = EmailField("Email", [validators.Length(min=6, max=35),
+        validators.Required(lazy_gettext("Please enter your email address."))])
+    password = PasswordField(lazy_gettext('Password'),
+            [validators.Required(lazy_gettext("Please enter a password.")),
+             validators.Length(min=6, max=100)])
     submit = SubmitField(lazy_gettext("Log In"))
 
-    def __init__(self, *args, **kwargs):
-        Form.__init__(self, *args, **kwargs)
-
     def validate(self):
-        if not Form.validate(self):
+        if not super(SigninForm, self).validate():
             return False
 
         user = User.query.filter(User.email == self.email.data).first()
-        if user and user.check_password(self.password.data) and user.activation_key == "":
+        if user and user.check_password(self.password.data) \
+                and user.activation_key == "":
             return True
         elif user and user.activation_key != "":
             flash(lazy_gettext('Account not confirmed'), 'danger')
@@ -79,6 +84,7 @@ class SigninForm(Form):
             #self.email.errors.append("Invalid email or password")
             return False
 
+
 class AddFeedForm(Form):
     title = TextField(lazy_gettext("Title"), [validators.Optional()])
     link = TextField(lazy_gettext("Feed link"), [validators.Optional()])
@@ -86,45 +92,52 @@ class AddFeedForm(Form):
     enabled = BooleanField(lazy_gettext("Check for updates"), default=True)
     submit = SubmitField(lazy_gettext("Save"))
 
-    def __init__(self, *args, **kwargs):
-        Form.__init__(self, *args, **kwargs)
-
-    def validate(self):
-        if not Form.validate(self):
-            return False
-        return True
 
 class ProfileForm(Form):
-    nickname = TextField(lazy_gettext("Nickname"), [validators.Required(lazy_gettext("Please enter your nickname."))])
-    email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email."))])
+    nickname = TextField(lazy_gettext("Nickname"),
+            [validators.Required(lazy_gettext("Please enter your nickname."))])
+    email = EmailField(lazy_gettext("Email"),
+               [validators.Length(min=6, max=35),
+                validators.Required(lazy_gettext("Please enter your email."))])
     password = PasswordField(lazy_gettext("Password"))
+    password_conf = PasswordField(lazy_gettext("Password Confirmation"))
+    refresh_rate = IntegerField(lazy_gettext("Feeds refresh frequency "
+                                             "(in minutes)"),
+                                default=60)
     submit = SubmitField(lazy_gettext("Save"))
 
-    def __init__(self, *args, **kwargs):
-        Form.__init__(self, *args, **kwargs)
-
     def validate(self):
-        if not Form.validate(self):
-            return False
+        validated = super(ProfileForm, self).validate()
+        if self.password.data != self.password_conf.data:
+            message = lazy_gettext("Passwords aren't the same.")
+            self.password.errors.append(message)
+            self.password_conf.errors.append(message)
+            validated = False
         if self.nickname.data != User.make_valid_nickname(self.nickname.data):
-            self.nickname.errors.append(lazy_gettext('This nickname has invalid characters. Please use letters, numbers, dots and underscores only.'))
-            return False
-        return True
+            self.nickname.errors.append(lazy_gettext('This nickname has '
+                    'invalid characters. Please use letters, numbers, dots and'
+                    ' underscores only.'))
+            validated = False
+        return validated
+
 
 class InformationMessageForm(Form):
-    subject = TextField(lazy_gettext("Subject"), [validators.Required(lazy_gettext("Please enter a subject."))])
-    message = TextAreaField(lazy_gettext("Message"), [validators.Required(lazy_gettext("Please enter a content."))])
+    subject = TextField(lazy_gettext("Subject"),
+            [validators.Required(lazy_gettext("Please enter a subject."))])
+    message = TextAreaField(lazy_gettext("Message"),
+            [validators.Required(lazy_gettext("Please enter a content."))])
     submit = SubmitField(lazy_gettext("Send"))
 
+
 class RecoverPasswordForm(Form):
-    email = EmailField(lazy_gettext("Email"), [validators.Length(min=6, max=35), validators.Required(lazy_gettext("Please enter your email address."))])
+    email = EmailField(lazy_gettext("Email"),
+            [validators.Length(min=6, max=35),
+             validators.Required(
+                 lazy_gettext("Please enter your email address."))])
     submit = SubmitField(lazy_gettext("Recover"))
 
-    def __init__(self, *args, **kwargs):
-        Form.__init__(self, *args, **kwargs)
-
     def validate(self):
-        if not Form.validate(self):
+        if not super(RecoverPasswordForm, self).validate():
             return False
 
         user = User.query.filter(User.email == self.email.data).first()
diff --git a/pyaggr3g470r/lib/__init__.py b/pyaggr3g470r/lib/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/pyaggr3g470r/lib/__init__.py
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
new file mode 100644
index 00000000..64ef8b6d
--- /dev/null
+++ b/pyaggr3g470r/lib/crawler.py
@@ -0,0 +1,257 @@
+"""
+Here's a sum up on how it works :
+
+CrawlerScheduler.run
+    will retreive a list of feeds to be refreshed and pass result to
+CrawlerScheduler.callback
+    which will retreive each feed and treat result with
+FeedCrawler.callback
+    which will interprete the result (status_code, etag) collect ids
+    and match them agaisnt pyagg which will cause
+PyAggUpdater.callback
+    to create the missing entries
+"""
+
+import time
+import conf
+import json
+import logging
+import requests
+import feedparser
+import dateutil.parser
+from functools import wraps
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor
+from requests_futures.sessions import FuturesSession
+from pyaggr3g470r.lib.utils import default_handler
+
+logger = logging.getLogger(__name__)
+API_ROOT = "api/v2.0/"
+
+
+def extract_id(entry, keys=[('link', 'link'),
+                            ('published', 'retrieved_date'),
+                            ('updated', 'retrieved_date')], force_id=False):
+    """For a given entry will return a dict that allows to identify it. The
+    dict will be constructed on the uid of the entry. if that identifier is
+    absent, the dict will be constructed upon the values of "keys".
+    """
+    entry_id = entry.get('entry_id') or entry.get('id')
+    if entry_id:
+        return {'entry_id': entry_id}
+    if not entry_id and force_id:
+        entry_id = hash("".join(entry[entry_key] for _, entry_key in keys
+                                if entry_key in entry))
+    else:
+        ids = {}
+        for entry_key, pyagg_key in keys:
+            if entry_key in entry and pyagg_key not in ids:
+                ids[pyagg_key] = entry[entry_key]
+                if 'date' in pyagg_key:
+                    ids[pyagg_key] = dateutil.parser.parse(ids[pyagg_key])\
+                                                    .isoformat()
+        return ids
+
+
+class AbstractCrawler:
+    __session__ = None
+    __counter__ = 0
+
+    def __init__(self, auth):
+        self.auth = auth
+        self.session = self.get_session()
+        self.url = conf.PLATFORM_URL
+
+    @classmethod
+    def get_session(cls):
+        """methods that allows us to treat session as a singleton"""
+        if cls.__session__ is None:
+            cls.__session__ = FuturesSession(
+                    executor=ThreadPoolExecutor(max_workers=conf.NB_WORKER))
+            cls.__session__.verify = False
+        return cls.__session__
+
+    @classmethod
+    def count_on_me(cls, func):
+        """A basic decorator which will count +1 at the begining of a call
+        and -1 at the end. It kinda allows us to wait for the __counter__ value
+        to be 0, meaning nothing is done anymore."""
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            cls.__counter__ += 1
+            result = func(*args, **kwargs)
+            cls.__counter__ -= 1
+            return result
+        return wrapper
+
+    def query_pyagg(self, method, urn, data=None):
+        """A wrapper for internal call, method should be ones you can find
+        on requests (header, post, get, options, ...), urn the distant
+        resources you want to access on pyagg, and data, the data you wanna
+        transmit."""
+        if data is None:
+            data = {}
+        method = getattr(self.session, method)
+        return method("%s%s%s" % (self.url, API_ROOT, urn),
+                      auth=self.auth, data=json.dumps(data,
+                                                      default=default_handler),
+                      headers={'Content-Type': 'application/json'})
+
+    @classmethod
+    def wait(cls):
+        "See count_on_me, that method will just wait for the counter to be 0"
+        time.sleep(1)
+        while cls.__counter__:
+            time.sleep(1)
+
+
+class PyAggUpdater(AbstractCrawler):
+
+    def __init__(self, feed, entries, headers, auth):
+        self.feed = feed
+        self.entries = entries
+        self.headers = headers
+        super(PyAggUpdater, self).__init__(auth)
+
+    def to_article(self, entry):
+        "Safe method to transorm a feedparser entry into an article"
+        date = datetime.now()
+
+        for date_key in ('published', 'updated'):
+            if entry.get(date_key):
+                try:
+                    date = dateutil.parser.parse(entry[date_key])
+                except Exception:
+                    pass
+                else:
+                    break
+        content = ''
+        if entry.get('content'):
+            content = entry['content'][0]['value']
+        elif entry.get('summary'):
+            content = entry['summary']
+
+        return {'feed_id': self.feed['id'],
+                'entry_id': extract_id(entry).get('entry_id', None),
+                'link': entry.get('link', self.feed['site_link']),
+                'title': entry.get('title', 'No title'),
+                'readed': False, 'like': False,
+                'content': content,
+                'retrieved_date': date.isoformat(),
+                'date': date.isoformat()}
+
+    @AbstractCrawler.count_on_me
+    def callback(self, response):
+        """Will process the result from the challenge, creating missing article
+        and updating the feed"""
+        results = response.result().json()
+        logger.debug('%r %r - %d entries were not matched and will be created',
+                     self.feed['id'], self.feed['title'], len(results))
+        for id_to_create in results:
+            entry = self.to_article(
+                    self.entries[tuple(sorted(id_to_create.items()))])
+            logger.info('creating %r - %r', entry['title'], id_to_create)
+            self.query_pyagg('post', 'article', entry)
+
+        now = datetime.now()
+        logger.debug('%r %r - updating feed etag %r last_mod %r',
+                     self.feed['id'], self.feed['title'],
+                     self.headers.get('etag'), now)
+
+        self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': 0,
+                     'etag': self.headers.get('etag', ''),
+                     'last_error': '',
+                     'last_modified': self.headers.get('last-modified', '')})
+
+
+class FeedCrawler(AbstractCrawler):
+
+    def __init__(self, feed, auth):
+        self.feed = feed
+        super(FeedCrawler, self).__init__(auth)
+
+    def clean_feed(self):
+        """Will reset the errors counters on a feed that have known errors"""
+        if self.feed.get('error_count') or self.feed.get('last_error'):
+            self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+                             {'error_count': 0, 'last_error': ''})
+
+    @AbstractCrawler.count_on_me
+    def callback(self, response):
+        """will fetch the feed and interprete results (304, etag) or will
+        challenge pyagg to compare gotten entries with existing ones"""
+        try:
+            response = response.result()
+            response.raise_for_status()
+        except Exception as error:
+            error_count = self.feed['error_count'] + 1
+            logger.warn('%r %r - an error occured while fetching feed; bumping'
+                        ' error count to %r', self.feed['id'],
+                        self.feed['title'], error_count)
+            self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+                             {'error_count': error_count,
+                              'last_error': str(error)})
+            return
+
+        if response.status_code == 304:
+            logger.info("%r %r - feed responded with 304",
+                         self.feed['id'], self.feed['title'])
+            self.clean_feed()
+            return
+        if self.feed['etag'] and response.headers.get('etag') \
+                and response.headers.get('etag') == self.feed['etag']:
+            logger.info("%r %r - feed responded with same etag (%d)",
+                         self.feed['id'], self.feed['title'],
+                         response.status_code)
+            self.clean_feed()
+            return
+        ids, entries = [], {}
+        parsed_response = feedparser.parse(response.text)
+        for entry in parsed_response['entries']:
+            entries[tuple(sorted(extract_id(entry).items()))] = entry
+            ids.append(extract_id(entry))
+        logger.debug('%r %r - found %d entries %r',
+                     self.feed['id'], self.feed['title'], len(ids), ids)
+        future = self.query_pyagg('get', 'articles/challenge', {'ids': ids})
+        updater = PyAggUpdater(self.feed, entries, response.headers, self.auth)
+        future.add_done_callback(updater.callback)
+
+
+class CrawlerScheduler(AbstractCrawler):
+
+    def __init__(self, username, password):
+        self.auth = (username, password)
+        super(CrawlerScheduler, self).__init__(self.auth)
+
+    def prepare_headers(self, feed):
+        """For a known feed, will construct some header dictionnary"""
+        headers = {}
+        if feed.get('etag', None):
+            headers['If-None-Match'] = feed['etag']
+        if feed.get('last_modified'):
+            headers['If-Modified-Since'] = feed['last_modified']
+        logger.debug('%r %r - calculated headers %r',
+                     feed['id'], feed['title'], headers)
+        return headers
+
+    @AbstractCrawler.count_on_me
+    def callback(self, response):
+        """processes feeds that need to be fetched"""
+        response = response.result()
+        response.raise_for_status()
+        feeds = response.json()
+        logger.debug('%d to fetch %r', len(feeds), feeds)
+        for feed in feeds:
+            logger.info('%r %r - fetching resources',
+                        feed['id'], feed['title'])
+            future = self.session.get(feed['link'],
+                                      headers=self.prepare_headers(feed))
+            future.add_done_callback(FeedCrawler(feed, self.auth).callback)
+
+    @AbstractCrawler.count_on_me
+    def run(self, **kwargs):
+        """entry point, will retreive feeds to be fetch
+        and launch the whole thing"""
+        logger.debug('retreving fetchable feed')
+        future = self.query_pyagg('get', 'feeds/fetchable', kwargs)
+        future.add_done_callback(self.callback)
diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py
new file mode 100644
index 00000000..a4f4b3ec
--- /dev/null
+++ b/pyaggr3g470r/lib/utils.py
@@ -0,0 +1,14 @@
+import types
+
+def default_handler(obj):
+    """JSON handler for default query formatting"""
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    if hasattr(obj, 'dump'):
+        return obj.dump()
+    if isinstance(obj, (set, frozenset, types.GeneratorType)):
+        return list(obj)
+    if isinstance(obj, BaseException):
+        return str(obj)
+    raise TypeError("Object of type %s with value of %r "
+                    "is not JSON serializable" % (type(obj), obj))
diff --git a/pyaggr3g470r/models.py b/pyaggr3g470r/models.py
deleted file mode 100644
index 8bc9e76b..00000000
--- a/pyaggr3g470r/models.py
+++ /dev/null
@@ -1,152 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# pyAggr3g470r - A Web based news aggregator.
-# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
-#
-# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 0.4 $"
-__date__ = "$Date: 2013/11/05 $"
-__revision__ = "$Date: 2014/04/12 $"
-__copyright__ = "Copyright (c) Cedric Bonhomme"
-__license__ = "GPLv3"
-
-import re
-import json
-import random, hashlib
-from datetime import datetime
-from sqlalchemy import asc, desc
-from werkzeug import generate_password_hash, check_password_hash
-from flask.ext.login import UserMixin
-
-from pyaggr3g470r import db
-
-class User(db.Model, UserMixin):
-    """
-    Represent a user.
-    """
-    id = db.Column(db.Integer, primary_key = True)
-    nickname = db.Column(db.String(), unique = True)
-    email = db.Column(db.String(254), index = True, unique = True)
-    pwdhash = db.Column(db.String())
-    roles = db.relationship('Role', backref = 'user', lazy = 'dynamic')
-    activation_key = db.Column(db.String(128), default =
-                               hashlib.sha512(
-                                   str(random.getrandbits(256)).encode("utf-8")
-                                   ).hexdigest()[:86])
-    date_created = db.Column(db.DateTime(), default=datetime.now)
-    last_seen = db.Column(db.DateTime(), default=datetime.now)
-    feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan')
-
-    @staticmethod
-    def make_valid_nickname(nickname):
-        return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
-
-    def get_id(self):
-        """
-        Return the id (email) of the user.
-        """
-        return self.email
-
-    def set_password(self, password):
-        """
-        Hash the password of the user.
-        """
-        self.pwdhash = generate_password_hash(password)
-
-    def check_password(self, password):
-        """
-        Check the password of the user.
-        """
-        return check_password_hash(self.pwdhash, password)
-
-    def is_admin(self):
-        """
-        Return True if the user has administrator rights.
-        """
-        return len([role for role in self.roles if role.name == "admin"]) != 0
-
-    def __eq__(self, other):
-        return self.id == other.id
-
-    def __repr__(self):
-        return '<User %r>' % (self.nickname)
-
-class Role(db.Model):
-    """
-    Represent a role.
-    """
-    id = db.Column(db.Integer, primary_key = True)
-    name = db.Column(db.String(), unique = True)
-
-    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-
-class Feed(db.Model):
-    """
-    Represent a feed.
-    """
-    id = db.Column(db.Integer, primary_key = True)
-    title = db.Column(db.String(), default="No title")
-    description = db.Column(db.String(), default="FR")
-    link = db.Column(db.String())
-    site_link = db.Column(db.String(), default="")
-    enabled = db.Column(db.Boolean(), default=True)
-    created_date = db.Column(db.DateTime(), default=datetime.now)
-    articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan',
-                                order_by=desc("Article.date"))
-
-    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-
-    def __repr__(self):
-        return '<Feed %r>' % (self.title)
-
-class Article(db.Model):
-    """
-    Represent an article from a feed.
-    """
-    id = db.Column(db.Integer, primary_key = True)
-    entry_id = db.Column(db.String())
-    link = db.Column(db.String())
-    title = db.Column(db.String())
-    content = db.Column(db.String())
-    readed = db.Column(db.Boolean(), default=False)
-    like = db.Column(db.Boolean(), default=False)
-    date = db.Column(db.DateTime(), default=datetime.now)
-    retrieved_date = db.Column(db.DateTime(), default=datetime.now)
-
-    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-    feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
-
-    def previous_article(self):
-        """
-        Returns the previous article (older).
-        """
-        return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first()
-
-    def next_article(self):
-        """
-        Returns the next article (newer).
-        """
-        return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first()
-
-    def __repr__(self):
-        return json.dumps({
-                            "title": self.title,
-                            "link": self.link,
-                            "content": self.content
-                         })
diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py
new file mode 100644
index 00000000..9584d1f2
--- /dev/null
+++ b/pyaggr3g470r/models/__init__.py
@@ -0,0 +1,35 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from .feed import Feed
+from .role import Role
+from .user import User
+from .article import Article
+
+
+__all__ = ['Feed', 'Role', 'User', 'Article']
diff --git a/pyaggr3g470r/models/article.py b/pyaggr3g470r/models/article.py
new file mode 100644
index 00000000..0466bc35
--- /dev/null
+++ b/pyaggr3g470r/models/article.py
@@ -0,0 +1,87 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+import json
+from datetime import datetime
+from flask import g
+from sqlalchemy import asc, desc
+
+db = g.db
+
+
+class Article(db.Model):
+    """
+    Represent an article from a feed.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    entry_id = db.Column(db.String())
+    link = db.Column(db.String())
+    title = db.Column(db.String())
+    content = db.Column(db.String())
+    readed = db.Column(db.Boolean(), default=False)
+    like = db.Column(db.Boolean(), default=False)
+    date = db.Column(db.DateTime(), default=datetime.now)
+    retrieved_date = db.Column(db.DateTime(), default=datetime.now)
+
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
+
+    def previous_article(self):
+        """
+        Returns the previous article (older).
+        """
+        return Article.query.filter(Article.date < self.date,
+                                    Article.feed_id == self.feed_id)\
+                            .order_by(desc("Article.date")).first()
+
+    def next_article(self):
+        """
+        Returns the next article (newer).
+        """
+        return Article.query.filter(Article.date > self.date,
+                                    Article.feed_id == self.feed_id)\
+                            .order_by(asc("Article.date")).first()
+
+    def __repr__(self):
+        return json.dumps({
+                            "title": self.title,
+                            "link": self.link,
+                            "content": self.content
+                         })
+
+    def dump(self):
+        return {"id": self.id,
+                "title": self.title,
+                "link": self.link,
+                "content": self.content,
+                "readed": self.readed,
+                "like": self.like,
+                "date": self.date,
+                "retrieved_date": self.retrieved_date,
+                "feed_id": getattr(self.source, 'id', None),
+                "feed_name": getattr(self.source, 'title', None)}
diff --git a/pyaggr3g470r/models/feed.py b/pyaggr3g470r/models/feed.py
new file mode 100644
index 00000000..28cb2b92
--- /dev/null
+++ b/pyaggr3g470r/models/feed.py
@@ -0,0 +1,75 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from datetime import datetime
+from flask import g
+from sqlalchemy import desc
+
+db = g.db
+
+
+class Feed(db.Model):
+    """
+    Represent a station.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    title = db.Column(db.String(), default="No title")
+    description = db.Column(db.String(), default="FR")
+    link = db.Column(db.String())
+    site_link = db.Column(db.String(), default="")
+    enabled = db.Column(db.Boolean(), default=True)
+    created_date = db.Column(db.DateTime(), default=datetime.now)
+
+    # cache handling
+    etag = db.Column(db.String(), default="")
+    last_modified = db.Column(db.String(), default="")
+    last_retreived = db.Column(db.DateTime(), default=datetime(1970, 1, 1))
+
+    # error logging
+    last_error = db.Column(db.String(), default="")
+    error_count = db.Column(db.Integer(), default=0)
+
+    # relationship
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
+    articles = db.relationship('Article', backref='source', lazy='dynamic',
+                               cascade='all,delete-orphan',
+                               order_by=desc("Article.date"))
+
+    def __repr__(self):
+        return '<Feed %r>' % (self.title)
+
+    def dump(self):
+        return {"id": self.id,
+                "title": self.title,
+                "description": self.description,
+                "link": self.link,
+                "site_link": self.site_link,
+                "etag": self.etag,
+                "error_count": self.error_count,
+                "last_modified": self.last_modified,
+                "last_retreived": self.last_retreived}
diff --git a/pyaggr3g470r/models/role.py b/pyaggr3g470r/models/role.py
new file mode 100644
index 00000000..71497caf
--- /dev/null
+++ b/pyaggr3g470r/models/role.py
@@ -0,0 +1,41 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+from flask import g
+
+db = g.db
+
+
+class Role(db.Model):
+    """
+    Represent a role.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(), unique=True)
+
+    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
diff --git a/pyaggr3g470r/models/user.py b/pyaggr3g470r/models/user.py
new file mode 100644
index 00000000..f2a268db
--- /dev/null
+++ b/pyaggr3g470r/models/user.py
@@ -0,0 +1,89 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - https://www.cedricbonhomme.org
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.4 $"
+__date__ = "$Date: 2013/11/05 $"
+__revision__ = "$Date: 2014/04/12 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+import re
+import random
+import hashlib
+from datetime import datetime
+from flask import g
+from werkzeug import generate_password_hash, check_password_hash
+from flask.ext.login import UserMixin
+
+db = g.db
+
+
+class User(db.Model, UserMixin):
+    """
+    Represent a user.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    nickname = db.Column(db.String(), unique=True)
+    email = db.Column(db.String(254), index=True, unique=True)
+    pwdhash = db.Column(db.String())
+    roles = db.relationship('Role', backref='user', lazy='dynamic')
+    activation_key = db.Column(db.String(128), default=hashlib.sha512(
+            str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86])
+    date_created = db.Column(db.DateTime(), default=datetime.now)
+    last_seen = db.Column(db.DateTime(), default=datetime.now)
+    feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic',
+                            cascade='all,delete-orphan')
+    refresh_rate = db.Column(db.Integer, default=60)  # in minutes
+
+    @staticmethod
+    def make_valid_nickname(nickname):
+        return re.sub('[^a-zA-Z0-9_\.]', '', nickname)
+
+    def get_id(self):
+        """
+        Return the id (email) of the user.
+        """
+        return self.email
+
+    def set_password(self, password):
+        """
+        Hash the password of the user.
+        """
+        self.pwdhash = generate_password_hash(password)
+
+    def check_password(self, password):
+        """
+        Check the password of the user.
+        """
+        return check_password_hash(self.pwdhash, password)
+
+    def is_admin(self):
+        """
+        Return True if the user has administrator rights.
+        """
+        return len([role for role in self.roles if role.name == "admin"]) != 0
+
+    def __eq__(self, other):
+        return self.id == other.id
+
+    def __repr__(self):
+        return '<User %r>' % (self.nickname)
diff --git a/pyaggr3g470r/notifications.py b/pyaggr3g470r/notifications.py
index 1acf782e..cf8fb723 100644
--- a/pyaggr3g470r/notifications.py
+++ b/pyaggr3g470r/notifications.py
@@ -19,9 +19,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from pyaggr3g470r import conf
+import conf
 from pyaggr3g470r import emails
 
+
 def information_message(subject, plaintext):
     """
     Send an information message to the users of the platform.
diff --git a/pyaggr3g470r/rest.py b/pyaggr3g470r/rest.py
deleted file mode 100644
index 1f354167..00000000
--- a/pyaggr3g470r/rest.py
+++ /dev/null
@@ -1,351 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# pyAggr3g470r - A Web based news aggregator.
-# Copyright (C) 2010-2015  Cédric Bonhomme - http://cedricbonhomme.org/
-#
-# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-__author__ = "Cedric Bonhomme"
-__version__ = "$Revision: 0.2 $"
-__date__ = "$Date: 2014/06/18 $"
-__revision__ = "$Date: 2014/07/05 $"
-__copyright__ = "Copyright (c) Cedric Bonhomme"
-__license__ = "AGPLv3"
-
-import re
-import dateutil.parser
-from functools import wraps
-from flask import g, Response, request, session, jsonify
-from flask.ext.restful import Resource, reqparse
-
-import conf
-if not conf.ON_HEROKU:
-    import pyaggr3g470r.search as fastsearch
-from pyaggr3g470r import api, db
-from pyaggr3g470r.models import User, Article, Feed
-
-def authenticate(func):
-    """
-    Decorator for the authentication to the web services.
-    """
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        if not getattr(func, 'authenticated', True):
-            return func(*args, **kwargs)
-
-        # authentication based on the session (already logged on the site)
-        if 'email' in session or g.user.is_authenticated():
-            return func(*args, **kwargs)
-
-        # authentication via HTTP only
-        auth = request.authorization
-        try:
-            email = auth.username
-            user = User.query.filter(User.email == email).first()
-            if user and user.check_password(auth.password) and user.activation_key == "":
-                g.user = user
-                return func(*args, **kwargs)
-        except AttributeError:
-            pass
-
-        return Response('<Authentication required>', 401,
-                        {'WWWAuthenticate':'Basic realm="Login Required"'})
-    return wrapper
-
-class ArticleListAPI(Resource):
-    """
-    Defines a RESTful API for Article elements.
-    """
-    method_decorators = [authenticate]
-
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        self.reqparse.add_argument('title', type = unicode, location = 'json')
-        self.reqparse.add_argument('content', type = unicode, location = 'json')
-        self.reqparse.add_argument('link', type = unicode, location = 'json')
-        self.reqparse.add_argument('date', type = str, location = 'json')
-        self.reqparse.add_argument('feed_id', type = int, location = 'json')
-        super(ArticleListAPI, self).__init__()
-
-    def get(self):
-        """
-        Returns a list of articles.
-        """
-        feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled}
-        articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), 
-                                        Article.user_id == g.user.id)
-        filter_ = request.args.get('filter_', 'unread')
-        feed_id = int(request.args.get('feed', 0))
-        limit = request.args.get('limit', 1000)
-        if filter_ != 'all':
-            articles = articles.filter(Article.readed == (filter_ == 'read'))
-        if feed_id:
-            articles = articles.filter(Article.feed_id == feed_id)
-
-        articles = articles.order_by(Article.date.desc())
-        if limit != 'all':
-            limit = int(limit)
-            articles = articles.limit(limit)
-
-        return jsonify(result= [{
-                                    "id": article.id,
-                                    "title": article.title,
-                                    "link": article.link,
-                                    "content": article.content,
-                                    "readed": article.readed,
-                                    "like": article.like,
-                                    "date": article.date,
-                                    "retrieved_date": article.retrieved_date,
-                                    "feed_id": article.source.id,
-                                    "feed_name": article.source.title
-                                }
-                                for article in articles]
-                        )
-
-    def post(self):
-        """
-        POST method - Create a new article.
-        """
-        args = self.reqparse.parse_args()
-        article_dict = {}
-        for k, v in args.iteritems():
-            if v != None:
-                article_dict[k] = v
-            else:
-                return {"message":"Missing argument: %s." % (k,)}
-        article_date = None
-        try:
-            article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True)
-        except:
-            try:  # trying to clean date field from letters
-                article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True))
-            except:
-                return jsonify({"message":"Bad format for the date."})
-        article = Article(link=article_dict["link"], title=article_dict["title"],
-                                content=article_dict["content"], readed=False, like=False,
-                                date=article_date, user_id=g.user.id,
-                                feed_id=article_dict["feed_id"])
-        feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first()
-        feed.articles.append(article)
-        try:
-            db.session.commit()
-            return jsonify({"message":"ok"})
-        except:
-            return jsonify({"message":"Impossible to create the article."})
-
-class ArticleAPI(Resource):
-    """
-    Defines a RESTful API for Article elements.
-    """
-    method_decorators = [authenticate]
-
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        self.reqparse.add_argument('like', type = bool, location = 'json')
-        self.reqparse.add_argument('readed', type = bool, location = 'json')
-        super(ArticleAPI, self).__init__()
-
-    def get(self, id=None):
-        """
-        Returns an article.
-        """
-        result = []
-        if id is not None:
-            article = Article.query.filter(Article.user_id == g.user.id, Article.id == id).first()
-            if article is not None:
-                if not article.readed:
-                    article.readed = True
-                    db.session.commit()
-                result.append(article)
-
-        return jsonify(result= [{
-                                    "id": article.id,
-                                    "title": article.title,
-                                    "link": article.link,
-                                    "content": article.content,
-                                    "readed": article.readed,
-                                    "like": article.like,
-                                    "date": article.date,
-                                    "retrieved_date": article.retrieved_date,
-                                    "feed_id": article.source.id,
-                                    "feed_name": article.source.title
-                                }
-                                for article in result]
-                        )
-
-    def put(self, id):
-        """
-        Update an article.
-        It is only possible to update the status ('like' and 'readed') of an article.
-        """
-        args = self.reqparse.parse_args()
-        article = Article.query.filter(Article.id == id).first()
-        if article is not None and article.source.subscriber.id == g.user.id:
-            if None is not args.get('like', None):
-                article.like = args['like']
-            if None is not args.get('readed', None):
-                article.readed = args['readed']
-            db.session.commit()
-
-            try:
-                fastsearch.delete_article(g.user.id, article.feed_id, article.id)
-            except:
-                pass
-
-            return jsonify({"message":"ok"})
-        else:
-            return jsonify({'message': 'Article not found.'})
-
-    def delete(self, id):
-        """
-        Delete an article.
-        """
-        article = Article.query.filter(Article.id == id).first()
-        if article is not None and article.source.subscriber.id == g.user.id:
-            db.session.delete(article)
-            db.session.commit()
-            return jsonify({"message":"ok"})
-        else:
-            return jsonify({'message': 'Article not found.'})
-
-api.add_resource(ArticleListAPI, '/api/v1.0/articles', endpoint = 'articles.json')
-api.add_resource(ArticleAPI, '/api/v1.0/articles/<int:id>', endpoint = 'article.json')
-
-class FeedListAPI(Resource):
-    """
-    Defines a RESTful API for Feed elements.
-    """
-    method_decorators = [authenticate]
-
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        self.reqparse.add_argument('title', type = unicode, default = "", location = 'json')
-        self.reqparse.add_argument('description', type = unicode, default = "", location = 'json')
-        self.reqparse.add_argument('link', type = unicode, location = 'json')
-        self.reqparse.add_argument('site_link', type = unicode, default = "", location = 'json')
-        self.reqparse.add_argument('enabled', type = bool, default = True ,location = 'json')
-        super(FeedListAPI, self).__init__()
-
-    def get(self):
-        """
-        Returns a list of feeds.
-        """
-        return jsonify(result= [{
-                                    "id": feed.id,
-                                    "title": feed.title,
-                                    "description": feed.description,
-                                    "link": feed.link,
-                                    "site_link": feed.site_link,
-                                    "enabled": feed.enabled,
-                                    "created_date": feed.created_date
-                                }
-                                for feed in g.user.feeds]
-                        )
-
-    def post(self):
-        """
-        POST method - Create a new feed.
-        """
-        args = self.reqparse.parse_args()
-        feed_dict = {}
-        for k, v in args.iteritems():
-            if v != None:
-                feed_dict[k] = v
-            else:
-                return jsonify({'message': 'missing argument: %s' % (k,)})
-        new_feed = Feed(title=feed_dict["title"], description=feed_dict["description"],
-                        link=feed_dict["link"], site_link=feed_dict["site_link"],
-                        enabled=feed_dict["enabled"])
-        g.user.feeds.append(new_feed)
-        try:
-            db.session.commit()
-            return jsonify({"message":"ok"})
-        except:
-            return jsonify({'message': 'Impossible to create the feed.'})
-
-class FeedAPI(Resource):
-    """
-    Defines a RESTful API for Feed elements.
-    """
-    method_decorators = [authenticate]
-
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        self.reqparse.add_argument('title', type = unicode, location = 'json')
-        self.reqparse.add_argument('description', type = unicode, location = 'json')
-        self.reqparse.add_argument('link', type = unicode, location = 'json')
-        self.reqparse.add_argument('site_link', type = unicode, location = 'json')
-        self.reqparse.add_argument('enabled', type = bool ,location = 'json')
-        super(FeedAPI, self).__init__()
-
-    def get(self, id=None):
-        """
-        Returns a feed.
-        """
-        result = []
-        if id is not None:
-            feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
-            if feed is not None:
-                result.append(feed)
-                return jsonify(result= [{
-                                            "id": feed.id,
-                                            "title": feed.title,
-                                            "description": feed.description,
-                                            "link": feed.link,
-                                            "site_link": feed.site_link,
-                                            "nb_articles": feed.articles.count()
-                                        }
-                                        for feed in result]
-                                )
-        return jsonify({'message': 'Feed not found'})
-
-    def put(self, id):
-        """
-        Update a feed.
-        """
-        args = self.reqparse.parse_args()
-        feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
-        if feed is not None:
-            if None is not args.get('title', None):
-                feed.title = args['title']
-            if None is not args.get('description', None):
-                feed.description = args['description']
-            if None is not args.get('link', None):
-                feed.link = args['link']
-            if None is not args.get('site_link', None):
-                feed.site_link = args['site_link']
-            if None is not args.get('enabled', None):
-                feed.enabled = args['enabled']
-            db.session.commit()
-            return jsonify({"message":"ok"})
-        else:
-            return jsonify({'message': 'Feed not found.'})
-
-    def delete(self, id):
-        """
-        Delete a feed.
-        """
-        feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first()
-        if feed is not None:
-            db.session.delete(feed)
-            db.session.commit()
-            return jsonify({"message":"ok"})
-        else:
-            return jsonify({'message': 'Feed not found.'})
-
-api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json')
-api.add_resource(FeedAPI, '/api/v1.0/feeds/<int:id>', endpoint = 'feed.json')
\ No newline at end of file
diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py
index 89fa0860..a7f780df 100644
--- a/pyaggr3g470r/search.py
+++ b/pyaggr3g470r/search.py
@@ -102,7 +102,7 @@ def delete_article(user_id, feed_id, article_id):
     try:
         ix = open_dir(indexdir)
     except (EmptyIndexError, OSError):
-        raise EmptyIndexError
+        return
     writer = ix.writer()
     document = And([Term("user_id", user_id), Term("feed_id", feed_id),
                     Term("article_id", article_id)])
diff --git a/pyaggr3g470r/static/js/articles.js b/pyaggr3g470r/static/js/articles.js
index 51273f3d..312a5cb6 100644
--- a/pyaggr3g470r/static/js/articles.js
+++ b/pyaggr3g470r/static/js/articles.js
@@ -18,6 +18,8 @@
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+API_ROOT = 'api/v2.0/'
+
 if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
 
 +function ($) {
@@ -78,7 +80,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
             // Encode your data as JSON.
             data: data,
             // This is the type of data you're expecting back from the server.
-            url: "/api/v1.0/articles/"+article_id,
+            url: API_ROOT + "article/" + article_id,
             success: function (result) {
                 //console.log(result);
             },
@@ -114,7 +116,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
             // Encode your data as JSON.
             data: data,
             // This is the type of data you're expecting back from the server.
-            url: "/api/v1.0/articles/"+article_id,
+            url: API_ROOT + "article/" + article_id,
             success: function (result) {
                 //console.log(result);
             },
@@ -132,7 +134,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') }
          // sends the updates to the server
         $.ajax({
             type: 'DELETE',
-            url: "/api/v1.0/articles/"+article_id,
+            url: API_ROOT + "article/" + article_id,
             success: function (result) {
                 //console.log(result);
             },
diff --git a/pyaggr3g470r/templates/edit_feed.html b/pyaggr3g470r/templates/edit_feed.html
index 1238e257..a6b28ded 100644
--- a/pyaggr3g470r/templates/edit_feed.html
+++ b/pyaggr3g470r/templates/edit_feed.html
@@ -17,7 +17,6 @@
 
             {{ form.enabled.label }}
             {{ form.enabled(class_="checkbox") }}
-
             <br />
             {{ form.submit(class_="btn btn-default") }}
         </form>
diff --git a/pyaggr3g470r/templates/feed.html b/pyaggr3g470r/templates/feed.html
index 21db7ebe..9910ccf7 100644
--- a/pyaggr3g470r/templates/feed.html
+++ b/pyaggr3g470r/templates/feed.html
@@ -14,6 +14,16 @@
             ({{ ((feed.articles.all()|count * 100 ) / nb_articles) | round(2, 'floor') }}% {{ _('of the database') }})
         {% endif %}
         .<br />
+        {% if feed.error_count > 2 %}
+            <b>{{ _("That feed has encountered too much consecutive errors and won't be retreived anymore") }}</b>
+        {% elif feed.error_count > 0 %}
+            {{ _("That feed has encountered some errors but that counter will be reinitialized at the next successful retreiving") }}
+        {% endif %}
+        .<br />
+        {% if feed.last_error %}
+            {{ _("Here's the last error encountered while retreiving this feed:") }} <pre>{{ feed.last_error }}</pre>
+        {% endif %}
+
         {{ _('Address of the feed') }}: <a href="{{ feed.link }}">{{ feed.link }}</a><br />
         {% if feed.site_link != "" %}
             {{ _('Address of the site') }}: <a href="{{ feed.site_link }}">{{ feed.site_link }}</a>
diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html
index a00f962e..8170a99d 100644
--- a/pyaggr3g470r/templates/home.html
+++ b/pyaggr3g470r/templates/home.html
@@ -21,6 +21,9 @@
         {% for fid, nbunread in unread|dictsort(by='value')|reverse %}
             <li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
                 {% if feed_id == fid %}<b>{% endif %}
+                    {% if in_error.get(fid, 0) > 0 %}
+                        <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span>
+                    {% endif %}
                     <span id="unread-{{ fid }}" class="badge pull-right">{{ nbunread }}</span>
                     {{ feeds[fid]|safe }}
                 {% if feed_id == fid %}</b>{% endif %}
@@ -36,6 +39,9 @@
         {% endfor %}
         {% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %}
             <li class="feed-menu"><a href="{{ gen_url(feed=fid) }}">
+                {% if in_error.get(fid, 0) > 0 %}
+                    <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span>
+                {% endif %}
                 {% if feed_id == fid %}<b>{% endif %}
                     {{ ftitle|safe }}
                 {% if feed_id == fid %}</b>{% endif %}
diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html
index 4dc62350..6b929bf3 100644
--- a/pyaggr3g470r/templates/layout.html
+++ b/pyaggr3g470r/templates/layout.html
@@ -7,9 +7,9 @@
     <meta name="description" content="pyAggr3g470r is a web-based news aggregator." />
     <meta name="author" content="" />
     <title>{% if head_title %}{{ head_title }} - {% endif %}pyAggr3g470r</title>
-    <link rel="shortcut icon" href="{{ url_for('.static', filename='img/favicon.png') }}" />
+    <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.png') }}" />
     <!-- Bootstrap core CSS -->
-    <link href="{{ url_for('.static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
+    <link href="{{ url_for('static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" />
     <!-- Add custom CSS here -->
     <style>
         body {
@@ -155,9 +155,9 @@
 
     <!-- Bootstrap core JavaScript -->
     <!-- Placed at the end of the document so the pages load faster -->
-    <script src="{{ url_for('.static', filename = 'js/jquery.js') }}"></script>
-    <script src="{{ url_for('.static', filename = 'js/bootstrap.js') }}"></script>
-    <script src="{{ url_for('.static', filename = 'js/articles.js') }}"></script>
+    <script src="{{ url_for('static', filename = 'js/jquery.js') }}"></script>
+    <script src="{{ url_for('static', filename = 'js/bootstrap.js') }}"></script>
+    <script src="{{ url_for('static', filename = 'js/articles.js') }}"></script>
     <script type="text/javascript" class="source">
         if (window.location.href.indexOf("filter_=all") > -1){
             $("#tab-all").attr('class', "active");
diff --git a/pyaggr3g470r/templates/profile.html b/pyaggr3g470r/templates/profile.html
index 6f22857c..f879b054 100644
--- a/pyaggr3g470r/templates/profile.html
+++ b/pyaggr3g470r/templates/profile.html
@@ -14,7 +14,10 @@
 
             {{ form.password.label }}
             {{ form.password(class_="form-control") }} {% for error in form.password.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
-
+            {{ form.password_conf.label }}
+            {{ form.password_conf(class_="form-control") }} {% for error in form.password_conf.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
+            {{ form.refresh_rate.label }}
+            {{ form.refresh_rate(class_="form-control") }} {% for error in form.refresh_rate.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
             <br />
             {{ form.submit(class_="btn") }}
         </form>
diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py
index 972909af..1fc84ff4 100755
--- a/pyaggr3g470r/utils.py
+++ b/pyaggr3g470r/utils.py
@@ -51,9 +51,10 @@ from collections import Counter
 from contextlib import contextmanager
 
 import conf
-from pyaggr3g470r import db
+from flask import g
 from pyaggr3g470r.models import User, Feed, Article
 
+
 # regular expression to check URL
 url_finders = [
     re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \
@@ -135,7 +136,7 @@ def import_opml(email, opml_content):
         return nb
 
     nb = read(subscriptions)
-    db.session.commit()
+    g.db.session.commit()
     return nb
 
 def import_json(email, json_content):
@@ -158,7 +159,7 @@ def import_json(email, json_content):
                                     enabled=feed["enabled"])
         user.feeds.append(new_feed)
         nb_feeds += 1
-    db.session.commit()
+    g.db.session.commit()
 
     # Create articles
     for feed in json_account["result"]:
@@ -178,7 +179,7 @@ def import_json(email, json_content):
 
                     user_feed.articles.append(new_article)
                     nb_articles += 1
-    db.session.commit()
+    g.db.session.commit()
 
     return nb_feeds, nb_articles
 
diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py
new file mode 100644
index 00000000..029dcb7d
--- /dev/null
+++ b/pyaggr3g470r/views/__init__.py
@@ -0,0 +1,5 @@
+from .views import *
+from .api import *
+
+from .article import article_bp, articles_bp
+from .feed import feed_bp, feeds_bp
diff --git a/pyaggr3g470r/views/api/__init__.py b/pyaggr3g470r/views/api/__init__.py
new file mode 100644
index 00000000..e11cdd95
--- /dev/null
+++ b/pyaggr3g470r/views/api/__init__.py
@@ -0,0 +1,31 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2015  Cédric Bonhomme - http://cedricbonhomme.org/
+#
+# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 0.2 $"
+__date__ = "$Date: 2014/06/18 $"
+__revision__ = "$Date: 2014/07/05 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "AGPLv3"
+
+from pyaggr3g470r.views.api import article, feed
+
+__all__ = ['article', 'feed']
diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py
new file mode 100644
index 00000000..17881412
--- /dev/null
+++ b/pyaggr3g470r/views/api/article.py
@@ -0,0 +1,58 @@
+from flask import g
+import dateutil.parser
+
+from pyaggr3g470r.controllers import ArticleController
+from pyaggr3g470r.views.api.common import PyAggAbstractResource,\
+                                          PyAggResourceNew, \
+                                          PyAggResourceExisting, \
+                                          PyAggResourceMulti
+
+
+ARTICLE_ATTRS = {'feed_id': {'type': str},
+                 'entry_id': {'type': str},
+                 'link': {'type': str},
+                 'title': {'type': str},
+                 'readed': {'type': bool}, 'like': {'type': bool},
+                 'content': {'type': str},
+                 'date': {'type': str}, 'retrieved_date': {'type': str}}
+
+
+class ArticleNewAPI(PyAggResourceNew):
+    controller_cls = ArticleController
+    attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
+
+
+class ArticleAPI(PyAggResourceExisting):
+    controller_cls = ArticleController
+    attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
+
+
+class ArticlesAPI(PyAggResourceMulti):
+    controller_cls = ArticleController
+    attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
+
+
+class ArticlesChallenge(PyAggAbstractResource):
+    controller_cls = ArticleController
+    attrs = {'ids': {'type': list, 'default': []}}
+    to_date = ['date', 'retrieved_date']
+
+    def get(self):
+        parsed_args = self.reqparse_args()
+        for id_dict in parsed_args['ids']:
+            for key in self.to_date:
+                if key in id_dict:
+                    id_dict[key] = dateutil.parser.parse(id_dict[key])
+
+        return self.controller.challenge(parsed_args['ids'])
+
+
+g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
+g.api.add_resource(ArticleAPI, '/article/<int:obj_id>',
+                   endpoint='article.json')
+g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json')
+g.api.add_resource(ArticlesChallenge, '/articles/challenge',
+                   endpoint='articles_challenge.json')
diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py
new file mode 100644
index 00000000..4f703712
--- /dev/null
+++ b/pyaggr3g470r/views/api/common.py
@@ -0,0 +1,217 @@
+"""For a given resources, classes in the module intend to create the following
+routes :
+    GET resource/<id>
+        -> to retreive one
+    POST resource
+        -> to create one
+    PUT resource/<id>
+        -> to update one
+    DELETE resource/<id>
+        -> to delete one
+
+    GET resources
+        -> to retreive several
+    POST resources
+        -> to create several
+    PUT resources
+        -> to update several
+    DELETE resources
+        -> to delete several
+"""
+import json
+import logging
+import dateutil.parser
+from copy import deepcopy
+from functools import wraps
+from werkzeug.exceptions import Unauthorized, BadRequest
+from flask import request, g, session, Response
+from flask.ext.restful import Resource, reqparse
+
+from pyaggr3g470r.lib.utils import default_handler
+from pyaggr3g470r.models import User
+
+logger = logging.getLogger(__name__)
+
+
+def authenticate(func):
+    """
+    Decorator for the authentication to the web services.
+    """
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        logged_in = False
+        if not getattr(func, 'authenticated', True):
+            logged_in = True
+        # authentication based on the session (already logged on the site)
+        elif 'email' in session or g.user.is_authenticated():
+            logged_in = True
+        else:
+            # authentication via HTTP only
+            auth = request.authorization
+            user = User.query.filter(User.nickname == auth.username).first()
+            if user and user.check_password(auth.password) \
+                    and user.activation_key == "":
+                g.user = user
+                logged_in = True
+
+        if logged_in:
+            return func(*args, **kwargs)
+        raise Unauthorized({'WWWAuthenticate': 'Basic realm="Login Required"'})
+    return wrapper
+
+
+def to_response(func):
+    """Will cast results of func as a result, and try to extract
+    a status_code for the Response object"""
+    def wrapper(*args, **kwargs):
+        status_code = 200
+        result = func(*args, **kwargs)
+        if isinstance(result, Response):
+            return result
+        elif isinstance(result, tuple):
+            result, status_code = result
+        return Response(json.dumps(result, default=default_handler),
+                        status=status_code)
+    return wrapper
+
+
+class PyAggAbstractResource(Resource):
+    method_decorators = [authenticate, to_response]
+    attrs = {}
+    to_date = []  # list of fields to cast to datetime
+
+    def __init__(self, *args, **kwargs):
+        super(PyAggAbstractResource, self).__init__(*args, **kwargs)
+
+    @property
+    def controller(self):
+        return self.controller_cls(getattr(g.user, 'id', None))
+
+    def reqparse_args(self, req=None, strict=False, default=True, args=None):
+        """
+        strict: bool
+            if True will throw 400 error if args are defined and not in request
+        default: bool
+            if True, won't return defaults
+        args: dict
+            the args to parse, if None, self.attrs will be used
+        """
+        parser = reqparse.RequestParser()
+        for attr_name, attrs in (args or self.attrs).items():
+            if attrs.pop('force_default', False):
+                parser.add_argument(attr_name, location='json', **attrs)
+            elif not default and (not request.json
+                    or request.json and attr_name not in request.json):
+                continue
+            else:
+                parser.add_argument(attr_name, location='json', **attrs)
+        parsed = parser.parse_args(strict=strict) if req is None \
+                else parser.parse_args(req, strict=strict)
+        for field in self.to_date:
+            if parsed.get(field):
+                try:
+                    parsed[field] = dateutil.parser.parse(parsed[field])
+                except Exception:
+                    logger.exception('failed to parse %r', parsed[field])
+        return parsed
+
+
+class PyAggResourceNew(PyAggAbstractResource):
+
+    def post(self):
+        """Create a single new object"""
+        return self.controller.create(**self.reqparse_args()), 201
+
+
+class PyAggResourceExisting(PyAggAbstractResource):
+
+    def get(self, obj_id=None):
+        """Retreive a single object"""
+        return self.controller.get(id=obj_id)
+
+    def put(self, obj_id=None):
+        """update an object, new attrs should be passed in the payload"""
+        args = self.reqparse_args(default=False)
+        new_values = {key: args[key] for key in
+                      set(args).intersection(self.attrs)}
+        self.controller.update({'id': obj_id}, new_values)
+
+    def delete(self, obj_id=None):
+        """delete a object"""
+        self.controller.delete(obj_id)
+        return None, 204
+
+
+class PyAggResourceMulti(PyAggAbstractResource):
+
+    def get(self):
+        """retreive several objects. filters can be set in the payload on the
+        different fields of the object, and a limit can be set in there as well
+        """
+        if 'application/json' != request.headers.get('Content-Type'):
+            raise BadRequest("Content-Type must be application/json")
+        limit = request.json.pop('limit', 10)
+        if not limit:
+            return [res for res in self.controller.read(**request.json).all()]
+        return [res for res in self.controller.read(**request.json).limit(limit)]
+
+    def post(self):
+        """creating several objects. payload should be a list of dict.
+        """
+        if 'application/json' != request.headers.get('Content-Type'):
+            raise BadRequest("Content-Type must be application/json")
+        status = 201
+        results = []
+        for attrs in request.json:
+            try:
+                results.append(self.controller.create(**attrs).id)
+            except Exception as error:
+                status = 206
+                results.append(str(error))
+        # if no operation succeded, it's not partial anymore, returning err 500
+        if status == 206 and results.count('ok') == 0:
+            status = 500
+        return results, status
+
+    def put(self):
+        """creating several objects. payload should be:
+        >>> payload
+        [[obj_id1, {attr1: val1, attr2: val2}]
+         [obj_id2, {attr1: val1, attr2: val2}]]
+        """
+        if 'application/json' != request.headers.get('Content-Type'):
+            raise BadRequest("Content-Type must be application/json")
+        status = 200
+        results = []
+        for obj_id, attrs in request.json:
+            try:
+                new_values = {key: attrs[key] for key in
+                              set(attrs).intersection(self.attrs)}
+                self.controller.update({'id': obj_id}, new_values)
+                results.append('ok')
+            except Exception as error:
+                status = 206
+                results.append(str(error))
+        # if no operation succeded, it's not partial anymore, returning err 500
+        if status == 206 and results.count('ok') == 0:
+            status = 500
+        return results, status
+
+    def delete(self):
+        """will delete several objects,
+        a list of their ids should be in the payload"""
+        if 'application/json' != request.headers.get('Content-Type'):
+            raise BadRequest("Content-Type must be application/json")
+        status = 204
+        results = []
+        for obj_id in request.json:
+            try:
+                self.controller.delete(obj_id)
+                results.append('ok')
+            except Exception as error:
+                status = 206
+                results.append(error)
+        # if no operation succeded, it's not partial anymore, returning err 500
+        if status == 206 and results.count('ok') == 0:
+            status = 500
+        return results, status
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
new file mode 100644
index 00000000..898e30b0
--- /dev/null
+++ b/pyaggr3g470r/views/api/feed.py
@@ -0,0 +1,57 @@
+from flask import g
+
+from pyaggr3g470r.controllers.feed import FeedController, \
+                                          DEFAULT_MAX_ERROR, DEFAULT_LIMIT
+
+from pyaggr3g470r.views.api.common import PyAggAbstractResource, \
+                                          PyAggResourceNew, \
+                                          PyAggResourceExisting, \
+                                          PyAggResourceMulti
+
+
+FEED_ATTRS = {'title': {'type': str},
+              'description': {'type': str},
+              'link': {'type': str},
+              'site_link': {'type': str},
+              'enabled': {'type': bool, 'default': True},
+              'etag': {'type': str, 'default': ''},
+              'last_modified': {'type': str},
+              'last_retreived': {'type': str},
+              'last_error': {'type': str},
+              'error_count': {'type': int, 'default': 0}}
+
+
+class FeedNewAPI(PyAggResourceNew):
+    controller_cls = FeedController
+    attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
+
+
+class FeedAPI(PyAggResourceExisting):
+    controller_cls = FeedController
+    attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
+
+
+class FeedsAPI(PyAggResourceMulti):
+    controller_cls = FeedController
+    attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
+
+
+class FetchableFeedAPI(PyAggAbstractResource):
+    controller_cls = FeedController
+    to_date = ['date', 'last_retreived']
+    attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR},
+             'limit': {'type': int, 'default': DEFAULT_LIMIT}}
+
+    def get(self):
+        return [feed for feed in self.controller.list_fetchable(
+                                                    **self.reqparse_args())]
+
+
+g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')
+g.api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json')
+g.api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json')
+g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable',
+                   endpoint='fetchable_feed.json')
diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py
new file mode 100644
index 00000000..66cc0f37
--- /dev/null
+++ b/pyaggr3g470r/views/article.py
@@ -0,0 +1,53 @@
+from flask import Blueprint, g, render_template, redirect
+from sqlalchemy import desc
+
+from pyaggr3g470r import controllers, utils
+from pyaggr3g470r.decorators import pyagg_default_decorator
+
+articles_bp = Blueprint('articles', __name__, url_prefix='/articles')
+article_bp = Blueprint('article', __name__, url_prefix='/article')
+
+
+@articles_bp.route('/<feed_id>', methods=['GET'])
+@articles_bp.route('/<feed_id>/<int:nb_articles>', methods=['GET'])
+@pyagg_default_decorator
+def articles(feed_id=None, nb_articles=-1):
+    """List articles of a feed. The administrator of the platform is able to
+    access to this view for every users."""
+    feed = controllers.FeedController(g.user.id).get(id=feed_id)
+    feed.articles = controllers.ArticleController(g.user.id)\
+                               .read(feed_id=feed.id)\
+                               .order_by(desc("Article.date"))
+    if len(feed.articles.all()) <= nb_articles:
+        nb_articles = -1
+    if nb_articles == -1:
+        feed.articles = feed.articles.limit(nb_articles)
+    return render_template('articles.html', feed=feed, nb_articles=nb_articles)
+
+
+@article_bp.route('/redirect/<int:article_id>', methods=['GET'])
+@pyagg_default_decorator
+def redirect_to_article(article_id):
+    article = controllers.ArticleController(g.user.id).get(id=article_id)
+    return redirect(article.link)
+
+
+@article_bp.route('/<int:article_id>', methods=['GET'])
+@pyagg_default_decorator
+def article(article_id=None):
+    """
+    Presents the content of an article.
+    """
+    article = controllers.ArticleController(g.user.id).get(id=article_id)
+    previous_article = article.previous_article()
+    if previous_article is None:
+        previous_article = article.source.articles[0]
+    next_article = article.next_article()
+    if next_article is None:
+        next_article = article.source.articles[-1]
+
+    return render_template('article.html',
+                           head_title=utils.clear_string(article.title),
+                           article=article,
+                           previous_article=previous_article,
+                           next_article=next_article)
diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py
new file mode 100644
index 00000000..2af502a7
--- /dev/null
+++ b/pyaggr3g470r/views/feed.py
@@ -0,0 +1,50 @@
+from datetime import datetime
+from flask import Blueprint, g, render_template
+
+from pyaggr3g470r import controllers, utils
+from pyaggr3g470r.decorators import pyagg_default_decorator, \
+                                    feed_access_required
+
+feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
+feed_bp = Blueprint('feed', __name__, url_prefix='/feed')
+
+@feeds_bp.route('/', methods=['GET'])
+def feeds():
+    "Lists the subscribed  feeds in a table."
+    return render_template('feeds.html',
+            feeds=controllers.FeedController(g.user.id).read())
+
+
+@feed_bp.route('/<int:feed_id>', methods=['GET'])
+@pyagg_default_decorator
+@feed_access_required
+def feed(feed_id=None):
+    "Presents detailed information about a feed."
+    feed = controllers.FeedController(g.user.id).get(id=feed_id)
+    word_size = 6
+    articles = controllers.ArticleController(g.user.id)\
+                          .read(feed_id=feed_id).all()
+    nb_articles = controllers.ArticleController(g.user.id).read().count()
+    top_words = utils.top_words(articles, n=50, size=int(word_size))
+    tag_cloud = utils.tag_cloud(top_words)
+
+    today = datetime.now()
+    try:
+        last_article = articles[0].date
+        first_article = articles[-1].date
+        delta = last_article - first_article
+        average = round(float(len(articles)) / abs(delta.days), 2)
+    except:
+        last_article = datetime.fromtimestamp(0)
+        first_article = datetime.fromtimestamp(0)
+        delta = last_article - first_article
+        average = 0
+    elapsed = today - last_article
+
+    return render_template('feed.html',
+                           head_title=utils.clear_string(feed.title),
+                           feed=feed, tag_cloud=tag_cloud,
+                           first_post_date=first_article,
+                           end_post_date=last_article,
+                           nb_articles=nb_articles,
+                           average=average, delta=delta, elapsed=elapsed)
diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views/views.py
index e5e07cde..5279d7fe 100644
--- a/pyaggr3g470r/views.py
+++ b/pyaggr3g470r/views/views.py
@@ -27,9 +27,14 @@ __copyright__ = "Copyright (c) Cedric Bonhomme"
 __license__ = "AGPLv3"
 
 import os
+import json
+import string
+import random
+import hashlib
 import datetime
 from collections import namedtuple
-from flask import abort, render_template, request, flash, session, \
+from bootstrap import application as app, db
+from flask import render_template, request, flash, session, Response, \
                   url_for, redirect, g, current_app, make_response, jsonify
 from flask.ext.login import LoginManager, login_user, logout_user, \
                             login_required, current_user, AnonymousUserMixin
@@ -37,17 +42,17 @@ from flask.ext.principal import Principal, Identity, AnonymousIdentity, \
                                 identity_changed, identity_loaded, Permission,\
                                 RoleNeed, UserNeed
 from flask.ext.babel import gettext
-from sqlalchemy import desc, func, or_
+from sqlalchemy import func, or_
 from sqlalchemy.exc import IntegrityError
 from werkzeug import generate_password_hash
 
 import conf
 from pyaggr3g470r import utils, notifications, export, duplicate
-from pyaggr3g470r import app, db, allowed_file, babel
 from pyaggr3g470r.models import User, Feed, Article, Role
 from pyaggr3g470r.decorators import feed_access_required
 from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \
                     ProfileForm, InformationMessageForm, RecoverPasswordForm
+from pyaggr3g470r.controllers import FeedController
 if not conf.ON_HEROKU:
     import pyaggr3g470r.search as fastsearch
 
@@ -118,7 +123,7 @@ def redirect_url(default='home'):
             request.referrer or \
             url_for(default)
 
-@babel.localeselector
+@g.babel.localeselector
 def get_locale():
     """
     Called before each request to give us a chance to choose
@@ -126,7 +131,7 @@ def get_locale():
     """
     return request.accept_languages.best_match(conf.LANGUAGES.keys())
 
-@babel.timezoneselector
+@g.babel.timezoneselector
 def get_timezone():
     try:
         return conf.TIME_ZONE[get_locale()]
@@ -152,11 +157,13 @@ def login():
         login_user(user)
         g.user = user
         session['email'] = form.email.data
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(current_app._get_current_object(),
+                              identity=Identity(user.id))
         flash(gettext("Logged in successfully."), 'success')
         return redirect(url_for('home'))
     return render_template('login.html', form=form)
 
+
 @app.route('/logout')
 @login_required
 def logout():
@@ -240,23 +247,14 @@ def home():
     unread = db.session.query(Article.feed_id, func.count(Article.id))\
                        .filter(Article.readed == False, Article.user_id == g.user.id)\
                        .group_by(Article.feed_id).all()
+    in_error = {feed.id: feed.error_count for feed in
+                FeedController(g.user.id).read(error_count__gt=0).all()}
     def gen_url(filter_=filter_, limit=limit, feed=feed_id):
         return '?filter_=%s&limit=%s&feed=%d' % (filter_, limit, feed)
     return render_template('home.html', gen_url=gen_url, feed_id=feed_id,
                            filter_=filter_, limit=limit, feeds=feeds,
-                           unread=dict(unread), articles=articles.all())
-
-
-@app.route('/article/redirect/<int:article_id>', methods=['GET'])
-@login_required
-def redirect_to_article(article_id):
-    article = Article.query.filter(Article.id == article_id,
-                                   Article.user_id == g.user.id).first()
-    if article is None:
-        abort(404)
-    article.readed = True
-    db.session.commit()
-    return redirect(article.link)
+                           unread=dict(unread), articles=articles.all(),
+                           in_error=in_error)
 
 
 @app.route('/fetch', methods=['GET'])
@@ -278,71 +276,6 @@ def about():
     """
     return render_template('about.html')
 
-@app.route('/feeds', methods=['GET'])
-@login_required
-def feeds():
-    """
-    Lists the subscribed  feeds in a table.
-    """
-    user = User.query.filter(User.email == g.user.email).first()
-    return render_template('feeds.html', feeds=user.feeds)
-
-@app.route('/feed/<int:feed_id>', methods=['GET'])
-@login_required
-@feed_access_required
-def feed(feed_id=None):
-    """
-    Presents detailed information about a feed.
-    """
-    feed = Feed.query.filter(Feed.id == feed_id).first()
-    word_size = 6
-    articles = feed.articles.all()
-    nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all())
-    top_words = utils.top_words(articles, n=50, size=int(word_size))
-    tag_cloud = utils.tag_cloud(top_words)
-
-    today = datetime.datetime.now()
-    try:
-        last_article = articles[0].date
-        first_article = articles[-1].date
-        delta = last_article - first_article
-        average = round(float(len(articles)) / abs(delta.days), 2)
-    except:
-        last_article = datetime.datetime.fromtimestamp(0)
-        first_article = datetime.datetime.fromtimestamp(0)
-        delta = last_article - first_article
-        average = 0
-    elapsed = today - last_article
-
-    return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \
-                        first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \
-                        average=average, delta=delta, elapsed=elapsed)
-
-@app.route('/article/<int:article_id>', methods=['GET'])
-@login_required
-def article(article_id=None):
-    """
-    Presents the content of an article.
-    """
-    article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first()
-    if article is not None:
-        if not article.readed:
-            article.readed = True
-            db.session.commit()
-
-        previous_article = article.previous_article()
-        if previous_article is None:
-            previous_article = article.source.articles[0]
-        next_article = article.next_article()
-        if next_article is None:
-            next_article = article.source.articles[-1]
-
-        return render_template('article.html', head_title=utils.clear_string(article.title),
-                               article=article,
-                               previous_article=previous_article, next_article=next_article)
-    flash(gettext("This article do not exist."), 'warning')
-    return redirect(url_for('home'))
-
 
 @app.route('/mark_as/<string:new_value>', methods=['GET'])
 @app.route('/mark_as/<string:new_value>/feed/<int:feed_id>', methods=['GET'])
@@ -403,24 +336,6 @@ def delete(article_id=None):
         flash(gettext('This article do not exist.'), 'danger')
         return redirect(url_for('home'))
 
-@app.route('/articles/<feed_id>', methods=['GET'])
-@app.route('/articles/<feed_id>/<int:nb_articles>', methods=['GET'])
-@login_required
-@feed_access_required
-def articles(feed_id=None, nb_articles=-1):
-    """
-    List articles of a feed.
-    The administrator of the platform is able to access to this view for every users.
-    """
-    feed = Feed.query.filter(Feed.id == feed_id).first()
-    new_feed = feed
-    if len(feed.articles.all()) <= nb_articles:
-        nb_articles = -1
-    if nb_articles == -1:
-        nb_articles = int(1e9)
-    new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \
-                                        Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles)
-    return render_template('articles.html', feed=new_feed, nb_articles=nb_articles)
 
 @app.route('/favorites', methods=['GET'])
 @login_required
@@ -591,7 +506,7 @@ def management():
         if None != request.files.get('opmlfile', None):
             # Import an OPML file
             data = request.files.get('opmlfile', None)
-            if not allowed_file(data.filename):
+            if not g.allowed_file(data.filename):
                 flash(gettext('File not allowed.'), 'danger')
             else:
                 try:
@@ -604,7 +519,7 @@ def management():
         elif None != request.files.get('jsonfile', None):
             # Import an account
             data = request.files.get('jsonfile', None)
-            if not allowed_file(data.filename):
+            if not g.allowed_file(data.filename):
                 flash(gettext('File not allowed.'), 'danger')
             else:
                 try:
@@ -627,7 +542,7 @@ def management():
 @app.route('/history', methods=['GET'])
 @login_required
 def history():
-    user = User.query.filter(User.id == g.user.id).first()
+    #user = User.query.filter(User.id == g.user.id).first()
     return render_template('history.html')
 
 @app.route('/bookmarklet', methods=['GET'])
@@ -639,7 +554,7 @@ def edit_feed(feed_id=None):
     """
     Add or edit a feed.
     """
-    feed = Feed.query.filter(Feed.id == feed_id).first()
+    feed = FeedController(g.user.id).get(id=feed_id)
     form = AddFeedForm()
 
     if request.method == 'POST':
@@ -653,7 +568,7 @@ def edit_feed(feed_id=None):
             return redirect('/edit_feed/' + str(feed_id))
         else:
             # Create a new feed
-            existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data]
+            existing_feed = [f for f in g.user.feeds if feed.link == form.link.data]
             if len(existing_feed) == 0:
                 new_feed = Feed(title=form.title.data, description="", link=form.link.data, \
                                 site_link=form.site_link.data, enabled=form.enabled.data)
@@ -678,7 +593,7 @@ def edit_feed(feed_id=None):
 
         # Enable the user to add a feed with a bookmarklet
         if None is not request.args.get('url', None):
-            existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)]
+            existing_feed = [f for f in g.user.feeds if feed.link == request.args.get('url', None)]
             if len(existing_feed) == 0:
                 g.user.feeds.append(Feed(link=request.args.get('url', None)))
                 db.session.commit()
@@ -717,7 +632,9 @@ def profile():
             if form.password.data != "":
                 user.set_password(form.password.data)
             db.session.commit()
-            flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success')
+            flash("%s %s %s" % (gettext('User'), user.nickname,
+                                gettext('successfully updated.')),
+                  'success')
             return redirect(url_for('profile'))
         else:
             return render_template('profile.html', form=form)
@@ -777,8 +694,6 @@ def recover():
     Enables the user to recover its account when he has forgotten
     its password.
     """
-    import string
-    import random
     form = RecoverPasswordForm()
 
     if request.method == 'POST':
@@ -920,7 +835,6 @@ def disable_user(user_id=None):
                 flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger')
 
         else:
-            import random, hashlib
             user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86]
             flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success')
         db.session.commit()