diff options
Diffstat (limited to 'pyaggr3g470r')
| -rw-r--r-- | pyaggr3g470r/controllers/abstract.py | 2 | ||||
| -rw-r--r-- | pyaggr3g470r/controllers/article.py | 6 | ||||
| -rw-r--r-- | pyaggr3g470r/controllers/feed.py | 10 | ||||
| -rw-r--r-- | pyaggr3g470r/decorators.py | 5 | ||||
| -rw-r--r-- | pyaggr3g470r/emails.py | 1 | ||||
| -rwxr-xr-x | pyaggr3g470r/lib/client.py | 6 | ||||
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 11 | ||||
| -rw-r--r-- | pyaggr3g470r/models/__init__.py | 31 | ||||
| -rw-r--r-- | pyaggr3g470r/views/api/article.py | 107 | ||||
| -rw-r--r-- | pyaggr3g470r/views/api/common.py | 122 | ||||
| -rw-r--r-- | pyaggr3g470r/views/api/feed.py | 120 | ||||
| -rw-r--r-- | pyaggr3g470r/views/views.py | 2 |
12 files changed, 211 insertions, 212 deletions
diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py index 8960c3be..6fe45461 100644 --- a/pyaggr3g470r/controllers/abstract.py +++ b/pyaggr3g470r/controllers/abstract.py @@ -13,7 +13,7 @@ class AbstractController(object): if self.user_id: filters[self._user_id_key] = self.user_id db_filters = set() - for key, value in filters.iteritems(): + for key, value in filters.items(): if key.endswith('__gt'): db_filters.add(getattr(self._db_cls, key[:-4]) > value) elif key.endswith('__lt'): diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py index cfaf386d..0de223ee 100644 --- a/pyaggr3g470r/controllers/article.py +++ b/pyaggr3g470r/controllers/article.py @@ -6,10 +6,10 @@ from pyaggr3g470r.models import Article class ArticleController(AbstractController): _db_cls = Article - def read(self, obj_id): - article = super(ArticleController, self).read(obj_id) + def get(self, **filters): + article = super(ArticleController, self).read(**filters) if not article.readed: - self.update(obj_id, readed=True) + self.update(article.id, readed=True) return article def delete(self, obj_id): diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index ce1c413f..56cef997 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -16,8 +16,14 @@ class FeedController(AbstractController): max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60) feeds = [feed for feed in self.read(user_id=self.user_id, error_count__le=max_error, - last_refreshed__lt=max_last_refresh).limit(limit)] + last_modified=max_last_refresh).limit(limit)] self.update({'id__in': [feed.id for feed in feeds]}, - {'last_refreshed': now}) + {'last_modified': now}) return feeds + + def list_last_articles(self, feed_id, limit=50): + from pyaggr3g470r.controllers import ArticleController + return ArticleController(self.user_id)._get(feed_id=feed_id)\ + .order_by(ArticleController._db_cls.retrieved_date.desc())\ + .limit(limit) diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index a13dacac..9bae626d 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from threading import Thread from functools import wraps @@ -23,6 +23,7 @@ def async(f): thr.start() return wrapper + def feed_access_required(func): """ This decorator enables to check if a user has access to a feed. @@ -47,7 +48,7 @@ def handle_pyagg_error(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except PyAggError, error: + except PyAggError as error: flash(gettext(error.default_message), 'warning') return redirect(url_for('home')) return wrapper diff --git a/pyaggr3g470r/emails.py b/pyaggr3g470r/emails.py index 50f94761..9a129718 100644 --- a/pyaggr3g470r/emails.py +++ b/pyaggr3g470r/emails.py @@ -31,6 +31,7 @@ from pyaggr3g470r.decorators import async logger = logging.getLogger(__name__) + @async def send_async_email(mfrom, mto, msg): try: diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py index da6b1727..6b2fc9ae 100755 --- a/pyaggr3g470r/lib/client.py +++ b/pyaggr3g470r/lib/client.py @@ -1,16 +1,16 @@ #!/usr/bin/env python import json import requests -URL = 'domain.net' +import conf def get_client(email, password): client = requests.session() - client.get(URL + 'api/csrf', verify=False, + client.get(conf.PLATFORM_URL + 'api/csrf', verify=False, data=json.dumps({'email': email, 'password': password})) return client def get_articles(client): - return client.get(URL + 'api/v1.0/articles/').json + return client.get(conf.PLATFORM_URL + 'api/v1.0/articles/').json diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py new file mode 100644 index 00000000..1d7fca71 --- /dev/null +++ b/pyaggr3g470r/lib/crawler.py @@ -0,0 +1,11 @@ +import feedparser +import dateutil.parser.parse + + +def get_feed_content(feed): + etag = feed.get('etag', None) + last_modified = None + if feed.get('last_modified'): + last_modified = dateutil.parser.parse(feed['last_modified'])\ + .strftime('%a, %d %b %Y %H:%M:%S %Z') + return feedparser.parse(feed['link'], etag=etag, modified=last_modified) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py index f01eb3a3..27ee18b6 100644 --- a/pyaggr3g470r/models/__init__.py +++ b/pyaggr3g470r/models/__init__.py @@ -28,7 +28,8 @@ __license__ = "GPLv3" import re import json -import random, hashlib +import random +import hashlib from datetime import datetime from flask import g from sqlalchemy import asc, desc @@ -47,10 +48,8 @@ class User(db.Model, UserMixin): email = db.Column(db.String(254), index=True, unique=True) pwdhash = db.Column(db.String()) roles = db.relationship('Role', backref='user', lazy='dynamic') - activation_key = db.Column(db.String(128), default = - hashlib.sha512( - str(random.getrandbits(256)).encode("utf-8") - ).hexdigest()[:86]) + activation_key = db.Column(db.String(128), default=hashlib.sha512( + str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86]) date_created = db.Column(db.DateTime(), default=datetime.now) last_seen = db.Column(db.DateTime(), default=datetime.now) feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic', @@ -114,15 +113,21 @@ class Feed(db.Model): email_notification = db.Column(db.Boolean(), default=False) enabled = db.Column(db.Boolean(), default=True) created_date = db.Column(db.DateTime(), default=datetime.now) - last_refreshed = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # cache handling + etag = db.Column(db.String(), default="") + last_modified = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # error logging last_error = db.Column(db.String(), default="") error_count = db.Column(db.Integer(), default=0) + + # relationship + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) articles = db.relationship('Article', backref='source', lazy='dynamic', cascade='all,delete-orphan', order_by=desc("Article.date")) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - def __repr__(self): return '<Feed %r>' % (self.title) @@ -132,14 +137,15 @@ class Feed(db.Model): "description": self.description, "link": self.link, "site_link": self.site_link, - "nb_articles": self.articles.count()} + "etag": self.etag, + "last_modified": self.last_modified} class Article(db.Model): """ Represent an article from a feed. """ - id = db.Column(db.Integer, primary_key = True) + id = db.Column(db.Integer, primary_key=True) entry_id = db.Column(db.String()) link = db.Column(db.String()) title = db.Column(db.String()) @@ -148,6 +154,7 @@ class Article(db.Model): like = db.Column(db.Boolean(), default=False) date = db.Column(db.DateTime(), default=datetime.now) retrieved_date = db.Column(db.DateTime(), default=datetime.now) + guid = db.Column(db.String(), default="") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) @@ -174,6 +181,7 @@ class Article(db.Model): "link": self.link, "content": self.content }) + def dump(self): return {"id": self.id, "title": self.title, @@ -184,5 +192,4 @@ class Article(db.Model): "date": self.date, "retrieved_date": self.retrieved_date, "feed_id": self.source.id, - "feed_name": self.source.title, - } + "feed_name": self.source.title} diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index 3642cda9..ebda6247 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -1,97 +1,36 @@ -import re -import dateutil.parser +from flask import g -from flask import request, g -from flask.ext.restful import Resource, reqparse - -from pyaggr3g470r.models import Article, Feed from pyaggr3g470r.controllers import ArticleController -from pyaggr3g470r.views.api.common import authenticate, to_response, \ - PyAggResource - +from pyaggr3g470r.views.api.common import PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti -class ArticleListAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate, to_response] - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('content', type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('date', type=str, location='json') - self.reqparse.add_argument('feed_id', type=int, location='json') - super(ArticleListAPI, self).__init__() +ARTICLE_ATTRS = {'title': {'type': str}, + 'content': {'type': str}, + 'link': {'type': str}, + 'date': {'type': str}, + 'feed_id': {'type': int}, + 'like': {'type': bool}, + 'readed': {'type': bool}} - def get(self): - """ - Returns a list of articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) +class ArticleNewAPI(PyAggResourceNew): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS - return {'result': [article.dump() for article in articles]} - def post(self): - """ - POST method - Create a new article. - """ - args = self.reqparse.parse_args() - article_dict = {} - for k, v in args.iteritems(): - if v != None: - article_dict[k] = v - else: - return {"message": "Missing argument: %s." % (k,)}, 400 - article_date = None - try: - article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) - except: - try: # trying to clean date field from letters - article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) - except: - return {"message": "Bad format for the date."}, 400 - article = Article(link=article_dict["link"], title=article_dict["title"], - content=article_dict["content"], readed=False, like=False, - date=article_date, user_id=g.user.id, - feed_id=article_dict["feed_id"]) - feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() - feed.articles.append(article) - try: - g.db.session.commit() - return {"message": "ok"}, 201 - except: - return {"message": "Impossible to create the article."}, 500 +class ArticleAPI(PyAggResourceExisting): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS -class ArticleAPI(PyAggResource): - "Defines a RESTful API for Article elements." - method_decorators = [authenticate, to_response] +class ArticlesAPI(PyAggResourceMulti): controller_cls = ArticleController - editable_attrs = ['like', 'readed'] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('like', type=bool, location='json') - self.reqparse.add_argument('readed', type=bool, location= 'json') - super(ArticleAPI, self).__init__() + attrs = ARTICLE_ATTRS -g.api.add_resource(ArticleListAPI, '/articles', endpoint='articles.json') -g.api.add_resource(ArticleAPI, '/articles/<int:obj_id>', - endpoint='article.json') +g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') +g.api.add_resource(ArticleAPI, '/article/<int:obj_id>', + endpoint='article.json') +g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index edf560da..c0759c03 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -1,6 +1,8 @@ +import json +import types from functools import wraps -from flask import request, g, session, Response, jsonify -from flask.ext.restful import Resource +from flask import request, g, session, Response +from flask.ext.restful import Resource, reqparse from pyaggr3g470r.models import User from pyaggr3g470r.lib.exceptions import PyAggError @@ -35,42 +37,122 @@ def authenticate(func): return wrapper +def default_handler(obj): + """JSON handler for default query formatting""" + if hasattr(obj, 'isoformat'): + return obj.isoformat() + if hasattr(obj, 'dump'): + return obj.dump() + if isinstance(obj, (set, frozenset, types.GeneratorType)): + return list(obj) + raise TypeError("Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj)) + + def to_response(func): def wrapper(*args, **kwargs): try: - res = func(*args, **kwargs) - except PyAggError, error: - response = jsonify(**error.message) + result = func(*args, **kwargs) + except PyAggError as error: + response = Response(json.dumps(result[0], default=default_handler)) response.status_code = error.status_code return response - if isinstance(res, tuple): - response = jsonify(**res[0]) - if len(res) > 1: - response.status_code = res[1] - return response - return res + status_code = 200 + if isinstance(result, tuple): + result, status_code = result + response = Response(json.dumps(result, default=default_handler), + status=status_code) + return response return wrapper -class PyAggResource(Resource): +class PyAggAbstractResource(Resource): method_decorators = [authenticate, to_response] - controller_cls = None - editable_attrs = [] def __init__(self, *args, **kwargs): self.controller = self.controller_cls(g.user.id) - super(PyAggResource, self).__init__(*args, **kwargs) + super(PyAggAbstractResource, self).__init__(*args, **kwargs) + + def reqparse_args(self, strict=False, default=True): + """ + strict: bool + if True will throw 400 error if args are defined and not in request + default: bool + if True, won't return defaults + + """ + parser = reqparse.RequestParser() + for attr_name, attrs in self.attrs.items(): + if not default and attr_name not in request.args: + continue + parser.add_argument(attr_name, location='json', **attrs) + return parser.parse_args(strict=strict) + + +class PyAggResourceNew(PyAggAbstractResource): + + def post(self): + return self.controller.create(**self.reqparse_args()), 201 + + +class PyAggResourceExisting(PyAggAbstractResource): def get(self, obj_id=None): - return {'result': [self.controller.get(id=obj_id).dump()]} + return self.controller.get(id=obj_id).dump() def put(self, obj_id=None): - args = self.reqparse.parse_args() + args = self.reqparse_args() new_values = {key: args[key] for key in - set(args).intersection(self.editable_attrs)} + set(args).intersection(self.attrs)} self.controller.update(obj_id, **new_values) - return {"message": "ok"} def delete(self, obj_id=None): self.controller.delete(obj_id) - return {"message": "ok"}, 204 + return None, 204 + + +class PyAggResourceMulti(PyAggAbstractResource): + + def get(self): + filters = self.reqparse_args(default=False) + return [res.dump() for res in self.controller.read(**filters).all()] + + def post(self): + status = 201 + results = [] + args = [] # FIXME + for arg in args: + try: + results.append(self.controller.create(**arg).id) + except Exception as error: + status = 206 + results.append(error) + return results, status + + def put(self): + status = 200 + results = [] + args = {} # FIXME + for obj_id, attrs in args.items(): + try: + new_values = {key: args[key] for key in + set(attrs).intersection(self.editable_attrs)} + self.controller.update(obj_id, **new_values) + results.append('ok') + except Exception as error: + status = 206 + results.append(error) + return results, status + + def delete(self): + status = 204 + results = [] + obj_ids = [] # FIXME extract some real ids + for obj_id in obj_ids: + try: + self.controller.delete(obj_id) + results.append('ok') + except Exception as error: + status = 206 + results.append(error) + return results, status diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index 94a5a433..e6f74cfd 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,92 +1,42 @@ +from datetime import datetime from flask import g from flask.ext.restful import Resource, reqparse -from pyaggr3g470r.controllers.feed import FeedController, \ - DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.models import Feed +from pyaggr3g470r.controllers.feed import FeedController, \ + DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.views.api.common import authenticate, to_response, \ - PyAggResource +from pyaggr3g470r.views.api.common import PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti -class FeedListAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate, to_response] +FEED_ATTRS = {'title': {'type': str}, + 'description': {'type': str}, + 'link': {'type': str}, + 'site_link': {'type': str}, + 'email_notification': {'type': bool, 'default': False}, + 'enabled': {'type': bool, 'default': True}, + 'etag': {'type': str, 'default': None}, + 'last_modified': {'type': datetime}, + 'last_error': {'type': datetime}, + 'error_count': {'type': int, 'default': 0}} - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', - type=unicode, default="", location='json') - self.reqparse.add_argument('description', - type=unicode, default="", location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', - type=unicode, default="", location='json') - self.reqparse.add_argument('email_notification', - type=bool, default=False, location='json') - self.reqparse.add_argument('enabled', - type=bool, default=True, location='json') - super(FeedListAPI, self).__init__() - def get(self): - """ - Returns a list of feeds. - """ - return {'result': [{"id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "email_notification": feed.email_notification, - "enabled": feed.enabled, - "created_date": feed.created_date, - } for feed in g.user.feeds]} - - def post(self): - """ - POST method - Create a new feed. - """ - args = self.reqparse.parse_args() - feed_dict = {} - for k, v in args.iteritems(): - if v != None: - feed_dict[k] = v - else: - return {'message': 'missing argument: %s' % (k,)}, 400 - new_feed = Feed(title=feed_dict["title"], - description=feed_dict["description"], - link=feed_dict["link"], - site_link=feed_dict["site_link"], - email_notification=feed_dict["email_notification"], - enabled=feed_dict["enabled"]) - g.user.feeds.append(new_feed) - try: - g.db.session.commit() - return {"message": "ok"} - except: - return {'message': 'Impossible to create the feed.'}, 500 - - -class FeedAPI(PyAggResource): - "Defines a RESTful API for Feed elements." +class FeedNewAPI(PyAggResourceNew): controller_cls = FeedController - editable_attrs = ['title', 'description', 'link', 'site_link', - 'email_notification', 'enabled', 'last_refreshed', - 'last_error', 'error_count'] + attrs = FEED_ATTRS - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('description', - type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', type=unicode, location='json') - self.reqparse.add_argument('email_notification', - type=bool, location='json') - self.reqparse.add_argument('enabled', type=bool ,location='json') - super(FeedAPI, self).__init__() + +class FeedAPI(PyAggResourceExisting): + pass + controller_cls = FeedController + attrs = FEED_ATTRS + + +class FeedsAPI(PyAggResourceMulti): + pass + controller_cls = FeedController + attrs = FEED_ATTRS class FetchableFeedAPI(Resource): @@ -102,10 +52,12 @@ class FetchableFeedAPI(Resource): def get(self): args = self.reqparse.parse_args() controller = FeedController(g.user.id) - return {'result': [feed.dump() for feed in controller.list_fetchable( - max_error=args['max_error'], limit=args['limit'])]} + return [feed for feed in controller.list_fetchable( + max_error=args['max_error'], limit=args['limit'])] -g.api.add_resource(FeedListAPI, '/feeds', endpoint='feeds.json') -g.api.add_resource(FeedAPI, '/feeds/<int:obj_id>', endpoint='feed.json') -g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', endpoint='fetchable_feed.json') +g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') +g.api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json') +g.api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json') +g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', + endpoint='fetchable_feed.json') diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 543f69b7..053bb473 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -164,7 +164,7 @@ def login(): @app.route('/api/csrf', methods=['GET']) def get_csrf(): try: - data = json.loads(request.data) + data = json.loads(request.data.decode()) except ValueError: return Response(status=400) email = data.get('email') |