new crawler with cache control and error handling

3 files changed, 86 insertions, 66 deletions

diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py
index ebda6247..17881412 100644
--- a/pyaggr3g470r/views/api/article.py
+++ b/pyaggr3g470r/views/api/article.py
@@ -1,36 +1,58 @@
 from flask import g
+import dateutil.parser
 
 from pyaggr3g470r.controllers import ArticleController
-from pyaggr3g470r.views.api.common import PyAggResourceNew, \
+from pyaggr3g470r.views.api.common import PyAggAbstractResource,\
+                                          PyAggResourceNew, \
                                           PyAggResourceExisting, \
                                           PyAggResourceMulti
 
 
-ARTICLE_ATTRS = {'title': {'type': str},
-                 'content': {'type': str},
+ARTICLE_ATTRS = {'feed_id': {'type': str},
+                 'entry_id': {'type': str},
                  'link': {'type': str},
-                 'date': {'type': str},
-                 'feed_id': {'type': int},
-                 'like': {'type': bool},
-                 'readed': {'type': bool}}
+                 'title': {'type': str},
+                 'readed': {'type': bool}, 'like': {'type': bool},
+                 'content': {'type': str},
+                 'date': {'type': str}, 'retrieved_date': {'type': str}}
 
 
 class ArticleNewAPI(PyAggResourceNew):
     controller_cls = ArticleController
     attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
 
 
 class ArticleAPI(PyAggResourceExisting):
     controller_cls = ArticleController
     attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
 
 
 class ArticlesAPI(PyAggResourceMulti):
     controller_cls = ArticleController
     attrs = ARTICLE_ATTRS
+    to_date = ['date', 'retrieved_date']
+
+
+class ArticlesChallenge(PyAggAbstractResource):
+    controller_cls = ArticleController
+    attrs = {'ids': {'type': list, 'default': []}}
+    to_date = ['date', 'retrieved_date']
+
+    def get(self):
+        parsed_args = self.reqparse_args()
+        for id_dict in parsed_args['ids']:
+            for key in self.to_date:
+                if key in id_dict:
+                    id_dict[key] = dateutil.parser.parse(id_dict[key])
+
+        return self.controller.challenge(parsed_args['ids'])
 
 
 g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json')
 g.api.add_resource(ArticleAPI, '/article/<int:obj_id>',
                    endpoint='article.json')
 g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json')
+g.api.add_resource(ArticlesChallenge, '/articles/challenge',
+                   endpoint='articles_challenge.json')
diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py
index c0759c03..a9d35411 100644
--- a/pyaggr3g470r/views/api/common.py
+++ b/pyaggr3g470r/views/api/common.py
@@ -1,12 +1,16 @@
 import json
-import types
+import logging
+import dateutil.parser
 from functools import wraps
 from flask import request, g, session, Response
 from flask.ext.restful import Resource, reqparse
 
+from pyaggr3g470r.lib.utils import default_handler
 from pyaggr3g470r.models import User
 from pyaggr3g470r.lib.exceptions import PyAggError
 
+logger = logging.getLogger(__name__)
+
 
 def authenticate(func):
     """
@@ -24,55 +28,47 @@ def authenticate(func):
         # authentication via HTTP only
         auth = request.authorization
         try:
-            email = auth.username
-            user = User.query.filter(User.email == email).first()
-            if user and user.check_password(auth.password) and user.activation_key == "":
+            user = User.query.filter(User.nickname == auth.username).first()
+            if user and user.check_password(auth.password) \
+                    and user.activation_key == "":
                 g.user = user
-                return func(*args, **kwargs)
-        except AttributeError:
-            pass
-
-        return Response('<Authentication required>', 401,
-                        {'WWWAuthenticate':'Basic realm="Login Required"'})
+        except Exception:
+            return Response('<Authentication required>', 401,
+                            {'WWWAuthenticate':
+                                'Basic realm="Login Required"'})
+        return func(*args, **kwargs)
     return wrapper
 
 
-def default_handler(obj):
-    """JSON handler for default query formatting"""
-    if hasattr(obj, 'isoformat'):
-        return obj.isoformat()
-    if hasattr(obj, 'dump'):
-        return obj.dump()
-    if isinstance(obj, (set, frozenset, types.GeneratorType)):
-        return list(obj)
-    raise TypeError("Object of type %s with value of %r "
-                    "is not JSON serializable" % (type(obj), obj))
-
-
 def to_response(func):
     def wrapper(*args, **kwargs):
+        status_code = 200
         try:
             result = func(*args, **kwargs)
         except PyAggError as error:
-            response = Response(json.dumps(result[0], default=default_handler))
-            response.status_code = error.status_code
-            return response
-        status_code = 200
-        if isinstance(result, tuple):
-            result, status_code = result
-        response = Response(json.dumps(result, default=default_handler),
+            return Response(json.dumps(error, default=default_handler),
                             status=status_code)
-        return response
+        if isinstance(result, Response):
+            return result
+        elif isinstance(result, tuple):
+            result, status_code = result
+        return Response(json.dumps(result, default=default_handler),
+                        status=status_code)
     return wrapper
 
 
 class PyAggAbstractResource(Resource):
     method_decorators = [authenticate, to_response]
+    attrs = {}
+    to_date = []
 
     def __init__(self, *args, **kwargs):
-        self.controller = self.controller_cls(g.user.id)
         super(PyAggAbstractResource, self).__init__(*args, **kwargs)
 
+    @property
+    def controller(self):
+        return self.controller_cls(getattr(g.user, 'id', None))
+
     def reqparse_args(self, strict=False, default=True):
         """
         strict: bool
@@ -83,10 +79,17 @@ class PyAggAbstractResource(Resource):
         """
         parser = reqparse.RequestParser()
         for attr_name, attrs in self.attrs.items():
-            if not default and attr_name not in request.args:
+            if not default and attr_name not in request.json:
                 continue
             parser.add_argument(attr_name, location='json', **attrs)
-        return parser.parse_args(strict=strict)
+        parsed = parser.parse_args(strict=strict)
+        for field in self.to_date:
+            if parsed.get(field):
+                try:
+                    parsed[field] = dateutil.parser.parse(parsed[field])
+                except Exception:
+                    logger.exception('failed to parse %r', parsed[field])
+        return parsed
 
 
 class PyAggResourceNew(PyAggAbstractResource):
@@ -98,13 +101,13 @@ class PyAggResourceNew(PyAggAbstractResource):
 class PyAggResourceExisting(PyAggAbstractResource):
 
     def get(self, obj_id=None):
-        return self.controller.get(id=obj_id).dump()
+        return self.controller.get(id=obj_id)
 
     def put(self, obj_id=None):
-        args = self.reqparse_args()
+        args = self.reqparse_args(default=False)
         new_values = {key: args[key] for key in
                       set(args).intersection(self.attrs)}
-        self.controller.update(obj_id, **new_values)
+        self.controller.update({'id': obj_id}, new_values)
 
     def delete(self, obj_id=None):
         self.controller.delete(obj_id)
@@ -115,7 +118,7 @@ class PyAggResourceMulti(PyAggAbstractResource):
 
     def get(self):
         filters = self.reqparse_args(default=False)
-        return [res.dump() for res in self.controller.read(**filters).all()]
+        return [res for res in self.controller.read(**filters).all()]
 
     def post(self):
         status = 201
@@ -137,7 +140,7 @@ class PyAggResourceMulti(PyAggAbstractResource):
             try:
                 new_values = {key: args[key] for key in
                               set(attrs).intersection(self.editable_attrs)}
-                self.controller.update(obj_id, **new_values)
+                self.controller.update({'id': obj_id}, new_values)
                 results.append('ok')
             except Exception as error:
                 status = 206
diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py
index e6f74cfd..625ad52d 100644
--- a/pyaggr3g470r/views/api/feed.py
+++ b/pyaggr3g470r/views/api/feed.py
@@ -1,11 +1,10 @@
-from datetime import datetime
 from flask import g
-from flask.ext.restful import Resource, reqparse
 
 from pyaggr3g470r.controllers.feed import FeedController, \
                                           DEFAULT_MAX_ERROR, DEFAULT_LIMIT
 
-from pyaggr3g470r.views.api.common import PyAggResourceNew, \
+from pyaggr3g470r.views.api.common import PyAggAbstractResource, \
+                                          PyAggResourceNew, \
                                           PyAggResourceExisting, \
                                           PyAggResourceMulti
 
@@ -16,44 +15,40 @@ FEED_ATTRS = {'title': {'type': str},
               'site_link': {'type': str},
               'email_notification': {'type': bool, 'default': False},
               'enabled': {'type': bool, 'default': True},
-              'etag': {'type': str, 'default': None},
-              'last_modified': {'type': datetime},
-              'last_error': {'type': datetime},
+              'etag': {'type': str, 'default': ''},
+              'last_modified': {'type': str},
+              'last_retreived': {'type': str},
+              'last_error': {'type': str},
               'error_count': {'type': int, 'default': 0}}
 
 
 class FeedNewAPI(PyAggResourceNew):
     controller_cls = FeedController
     attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
 
 
 class FeedAPI(PyAggResourceExisting):
-    pass
     controller_cls = FeedController
     attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
 
 
 class FeedsAPI(PyAggResourceMulti):
-    pass
     controller_cls = FeedController
     attrs = FEED_ATTRS
+    to_date = ['date', 'last_retreived']
 
 
-class FetchableFeedAPI(Resource):
-
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        self.reqparse.add_argument('max_error', type=int, location='json',
-                default=DEFAULT_MAX_ERROR)
-        self.reqparse.add_argument('limit', type=int, location='json',
-                default=DEFAULT_LIMIT)
-        super(FetchableFeedAPI, self).__init__()
+class FetchableFeedAPI(PyAggAbstractResource):
+    controller_cls = FeedController
+    to_date = ['date', 'last_retreived']
+    attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR},
+             'limit': {'type': int, 'default': DEFAULT_LIMIT}}
 
     def get(self):
-        args = self.reqparse.parse_args()
-        controller = FeedController(g.user.id)
-        return [feed for feed in controller.list_fetchable(
-                            max_error=args['max_error'], limit=args['limit'])]
+        return [feed for feed in self.controller.list_fetchable(
+                                                    **self.reqparse_args())]
 
 
 g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json')