From 5572851eca3b2f1bc56aed7232284acc436d2f49 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Sun, 1 Mar 2015 03:20:12 +0100 Subject: new crawler with cache control and error handling --- pyaggr3g470r/views/api/article.py | 36 ++++++++++++++---- pyaggr3g470r/views/api/common.py | 79 ++++++++++++++++++++------------------- pyaggr3g470r/views/api/feed.py | 37 ++++++++---------- 3 files changed, 86 insertions(+), 66 deletions(-) (limited to 'pyaggr3g470r/views/api') diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index ebda6247..17881412 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -1,36 +1,58 @@ from flask import g +import dateutil.parser from pyaggr3g470r.controllers import ArticleController -from pyaggr3g470r.views.api.common import PyAggResourceNew, \ +from pyaggr3g470r.views.api.common import PyAggAbstractResource,\ + PyAggResourceNew, \ PyAggResourceExisting, \ PyAggResourceMulti -ARTICLE_ATTRS = {'title': {'type': str}, - 'content': {'type': str}, +ARTICLE_ATTRS = {'feed_id': {'type': str}, + 'entry_id': {'type': str}, 'link': {'type': str}, - 'date': {'type': str}, - 'feed_id': {'type': int}, - 'like': {'type': bool}, - 'readed': {'type': bool}} + 'title': {'type': str}, + 'readed': {'type': bool}, 'like': {'type': bool}, + 'content': {'type': str}, + 'date': {'type': str}, 'retrieved_date': {'type': str}} class ArticleNewAPI(PyAggResourceNew): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] class ArticleAPI(PyAggResourceExisting): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] class ArticlesAPI(PyAggResourceMulti): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] + + +class ArticlesChallenge(PyAggAbstractResource): + controller_cls = ArticleController + attrs = {'ids': {'type': list, 'default': []}} + to_date = ['date', 'retrieved_date'] + + def get(self): + parsed_args = self.reqparse_args() + for id_dict in parsed_args['ids']: + for key in self.to_date: + if key in id_dict: + id_dict[key] = dateutil.parser.parse(id_dict[key]) + + return self.controller.challenge(parsed_args['ids']) g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') g.api.add_resource(ArticleAPI, '/article/', endpoint='article.json') g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') +g.api.add_resource(ArticlesChallenge, '/articles/challenge', + endpoint='articles_challenge.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index c0759c03..a9d35411 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -1,12 +1,16 @@ import json -import types +import logging +import dateutil.parser from functools import wraps from flask import request, g, session, Response from flask.ext.restful import Resource, reqparse +from pyaggr3g470r.lib.utils import default_handler from pyaggr3g470r.models import User from pyaggr3g470r.lib.exceptions import PyAggError +logger = logging.getLogger(__name__) + def authenticate(func): """ @@ -24,55 +28,47 @@ def authenticate(func): # authentication via HTTP only auth = request.authorization try: - email = auth.username - user = User.query.filter(User.email == email).first() - if user and user.check_password(auth.password) and user.activation_key == "": + user = User.query.filter(User.nickname == auth.username).first() + if user and user.check_password(auth.password) \ + and user.activation_key == "": g.user = user - return func(*args, **kwargs) - except AttributeError: - pass - - return Response('', 401, - {'WWWAuthenticate':'Basic realm="Login Required"'}) + except Exception: + return Response('', 401, + {'WWWAuthenticate': + 'Basic realm="Login Required"'}) + return func(*args, **kwargs) return wrapper -def default_handler(obj): - """JSON handler for default query formatting""" - if hasattr(obj, 'isoformat'): - return obj.isoformat() - if hasattr(obj, 'dump'): - return obj.dump() - if isinstance(obj, (set, frozenset, types.GeneratorType)): - return list(obj) - raise TypeError("Object of type %s with value of %r " - "is not JSON serializable" % (type(obj), obj)) - - def to_response(func): def wrapper(*args, **kwargs): + status_code = 200 try: result = func(*args, **kwargs) except PyAggError as error: - response = Response(json.dumps(result[0], default=default_handler)) - response.status_code = error.status_code - return response - status_code = 200 - if isinstance(result, tuple): - result, status_code = result - response = Response(json.dumps(result, default=default_handler), + return Response(json.dumps(error, default=default_handler), status=status_code) - return response + if isinstance(result, Response): + return result + elif isinstance(result, tuple): + result, status_code = result + return Response(json.dumps(result, default=default_handler), + status=status_code) return wrapper class PyAggAbstractResource(Resource): method_decorators = [authenticate, to_response] + attrs = {} + to_date = [] def __init__(self, *args, **kwargs): - self.controller = self.controller_cls(g.user.id) super(PyAggAbstractResource, self).__init__(*args, **kwargs) + @property + def controller(self): + return self.controller_cls(getattr(g.user, 'id', None)) + def reqparse_args(self, strict=False, default=True): """ strict: bool @@ -83,10 +79,17 @@ class PyAggAbstractResource(Resource): """ parser = reqparse.RequestParser() for attr_name, attrs in self.attrs.items(): - if not default and attr_name not in request.args: + if not default and attr_name not in request.json: continue parser.add_argument(attr_name, location='json', **attrs) - return parser.parse_args(strict=strict) + parsed = parser.parse_args(strict=strict) + for field in self.to_date: + if parsed.get(field): + try: + parsed[field] = dateutil.parser.parse(parsed[field]) + except Exception: + logger.exception('failed to parse %r', parsed[field]) + return parsed class PyAggResourceNew(PyAggAbstractResource): @@ -98,13 +101,13 @@ class PyAggResourceNew(PyAggAbstractResource): class PyAggResourceExisting(PyAggAbstractResource): def get(self, obj_id=None): - return self.controller.get(id=obj_id).dump() + return self.controller.get(id=obj_id) def put(self, obj_id=None): - args = self.reqparse_args() + args = self.reqparse_args(default=False) new_values = {key: args[key] for key in set(args).intersection(self.attrs)} - self.controller.update(obj_id, **new_values) + self.controller.update({'id': obj_id}, new_values) def delete(self, obj_id=None): self.controller.delete(obj_id) @@ -115,7 +118,7 @@ class PyAggResourceMulti(PyAggAbstractResource): def get(self): filters = self.reqparse_args(default=False) - return [res.dump() for res in self.controller.read(**filters).all()] + return [res for res in self.controller.read(**filters).all()] def post(self): status = 201 @@ -137,7 +140,7 @@ class PyAggResourceMulti(PyAggAbstractResource): try: new_values = {key: args[key] for key in set(attrs).intersection(self.editable_attrs)} - self.controller.update(obj_id, **new_values) + self.controller.update({'id': obj_id}, new_values) results.append('ok') except Exception as error: status = 206 diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index e6f74cfd..625ad52d 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,11 +1,10 @@ -from datetime import datetime from flask import g -from flask.ext.restful import Resource, reqparse from pyaggr3g470r.controllers.feed import FeedController, \ DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.views.api.common import PyAggResourceNew, \ +from pyaggr3g470r.views.api.common import PyAggAbstractResource, \ + PyAggResourceNew, \ PyAggResourceExisting, \ PyAggResourceMulti @@ -16,44 +15,40 @@ FEED_ATTRS = {'title': {'type': str}, 'site_link': {'type': str}, 'email_notification': {'type': bool, 'default': False}, 'enabled': {'type': bool, 'default': True}, - 'etag': {'type': str, 'default': None}, - 'last_modified': {'type': datetime}, - 'last_error': {'type': datetime}, + 'etag': {'type': str, 'default': ''}, + 'last_modified': {'type': str}, + 'last_retreived': {'type': str}, + 'last_error': {'type': str}, 'error_count': {'type': int, 'default': 0}} class FeedNewAPI(PyAggResourceNew): controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] class FeedAPI(PyAggResourceExisting): - pass controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] class FeedsAPI(PyAggResourceMulti): - pass controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] -class FetchableFeedAPI(Resource): - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('max_error', type=int, location='json', - default=DEFAULT_MAX_ERROR) - self.reqparse.add_argument('limit', type=int, location='json', - default=DEFAULT_LIMIT) - super(FetchableFeedAPI, self).__init__() +class FetchableFeedAPI(PyAggAbstractResource): + controller_cls = FeedController + to_date = ['date', 'last_retreived'] + attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR}, + 'limit': {'type': int, 'default': DEFAULT_LIMIT}} def get(self): - args = self.reqparse.parse_args() - controller = FeedController(g.user.id) - return [feed for feed in controller.list_fetchable( - max_error=args['max_error'], limit=args['limit'])] + return [feed for feed in self.controller.list_fetchable( + **self.reqparse_args())] g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') -- cgit