From 4c14f36ab73f227bdd7ed636667d8035c33986e0 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Wed, 29 Oct 2014 14:32:24 +0100 Subject: global esthetics tweak --- pyaggr3g470r/__init__.py | 2 +- pyaggr3g470r/lib/__init__.py | 0 pyaggr3g470r/lib/client.py | 16 + pyaggr3g470r/models.py | 153 ------- pyaggr3g470r/models/__init__.py | 177 ++++++++ pyaggr3g470r/rest.py | 357 --------------- pyaggr3g470r/views.py | 930 -------------------------------------- pyaggr3g470r/views/__init__.py | 2 + pyaggr3g470r/views/api.py | 356 +++++++++++++++ pyaggr3g470r/views/views.py | 956 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 1508 insertions(+), 1441 deletions(-) create mode 100644 pyaggr3g470r/lib/__init__.py create mode 100755 pyaggr3g470r/lib/client.py delete mode 100644 pyaggr3g470r/models.py create mode 100644 pyaggr3g470r/models/__init__.py delete mode 100644 pyaggr3g470r/rest.py delete mode 100644 pyaggr3g470r/views.py create mode 100644 pyaggr3g470r/views/__init__.py create mode 100644 pyaggr3g470r/views/api.py create mode 100644 pyaggr3g470r/views/views.py diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index f3f784f4..4ba54095 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -45,4 +45,4 @@ app.jinja_env.filters['datetime'] = format_datetime from flask.ext.restful import Api api = Api(app) -from pyaggr3g470r import views, rest +from pyaggr3g470r import views diff --git a/pyaggr3g470r/lib/__init__.py b/pyaggr3g470r/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py new file mode 100755 index 00000000..da6b1727 --- /dev/null +++ b/pyaggr3g470r/lib/client.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import json +import requests +URL = 'domain.net' + + +def get_client(email, password): + client = requests.session() + client.get(URL + 'api/csrf', verify=False, + data=json.dumps({'email': email, + 'password': password})) + return client + + +def get_articles(client): + return client.get(URL + 'api/v1.0/articles/').json diff --git a/pyaggr3g470r/models.py b/pyaggr3g470r/models.py deleted file mode 100644 index b7a75d5f..00000000 --- a/pyaggr3g470r/models.py +++ /dev/null @@ -1,153 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import re -import json -import random, hashlib -from datetime import datetime -from sqlalchemy import asc, desc -from werkzeug import generate_password_hash, check_password_hash -from flask.ext.login import UserMixin - -from pyaggr3g470r import db - -class User(db.Model, UserMixin): - """ - Represent a user. - """ - id = db.Column(db.Integer, primary_key = True) - nickname = db.Column(db.String(), unique = True) - email = db.Column(db.String(254), index = True, unique = True) - pwdhash = db.Column(db.String()) - roles = db.relationship('Role', backref = 'user', lazy = 'dynamic') - activation_key = db.Column(db.String(128), default = - hashlib.sha512( - str(random.getrandbits(256)).encode("utf-8") - ).hexdigest()[:86]) - date_created = db.Column(db.DateTime(), default=datetime.now) - last_seen = db.Column(db.DateTime(), default=datetime.now) - feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan') - - @staticmethod - def make_valid_nickname(nickname): - return re.sub('[^a-zA-Z0-9_\.]', '', nickname) - - def get_id(self): - """ - Return the id (email) of the user. - """ - return self.email - - def set_password(self, password): - """ - Hash the password of the user. - """ - self.pwdhash = generate_password_hash(password) - - def check_password(self, password): - """ - Check the password of the user. - """ - return check_password_hash(self.pwdhash, password) - - def is_admin(self): - """ - Return True if the user has administrator rights. - """ - return len([role for role in self.roles if role.name == "admin"]) != 0 - - def __eq__(self, other): - return self.id == other.id - - def __repr__(self): - return '' % (self.nickname) - -class Role(db.Model): - """ - Represent a role. - """ - id = db.Column(db.Integer, primary_key = True) - name = db.Column(db.String(), unique = True) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - -class Feed(db.Model): - """ - Represent a station. - """ - id = db.Column(db.Integer, primary_key = True) - title = db.Column(db.String(), default="No title") - description = db.Column(db.String(), default="FR") - link = db.Column(db.String()) - site_link = db.Column(db.String(), default="") - email_notification = db.Column(db.Boolean(), default=False) - enabled = db.Column(db.Boolean(), default=True) - created_date = db.Column(db.DateTime(), default=datetime.now) - articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan', - order_by=desc("Article.date")) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - - def __repr__(self): - return '' % (self.title) - -class Article(db.Model): - """ - Represent an article from a feed. - """ - id = db.Column(db.Integer, primary_key = True) - entry_id = db.Column(db.String()) - link = db.Column(db.String()) - title = db.Column(db.String()) - content = db.Column(db.String()) - readed = db.Column(db.Boolean(), default=False) - like = db.Column(db.Boolean(), default=False) - date = db.Column(db.DateTime(), default=datetime.now) - retrieved_date = db.Column(db.DateTime(), default=datetime.now) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) - - def previous_article(self): - """ - Returns the previous article (older). - """ - return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first() - - def next_article(self): - """ - Returns the next article (newer). - """ - return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first() - - def __repr__(self): - return json.dumps({ - "title": self.title, - "link": self.link, - "content": self.content - }) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py new file mode 100644 index 00000000..2618ed73 --- /dev/null +++ b/pyaggr3g470r/models/__init__.py @@ -0,0 +1,177 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +import re +import json +import random, hashlib +from datetime import datetime +from sqlalchemy import asc, desc +from werkzeug import generate_password_hash, check_password_hash +from flask.ext.login import UserMixin + +from pyaggr3g470r import db + +class User(db.Model, UserMixin): + """ + Represent a user. + """ + id = db.Column(db.Integer, primary_key = True) + nickname = db.Column(db.String(), unique = True) + email = db.Column(db.String(254), index = True, unique = True) + pwdhash = db.Column(db.String()) + roles = db.relationship('Role', backref = 'user', lazy = 'dynamic') + activation_key = db.Column(db.String(128), default = + hashlib.sha512( + str(random.getrandbits(256)).encode("utf-8") + ).hexdigest()[:86]) + date_created = db.Column(db.DateTime(), default=datetime.now) + last_seen = db.Column(db.DateTime(), default=datetime.now) + feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan') + + @staticmethod + def make_valid_nickname(nickname): + return re.sub('[^a-zA-Z0-9_\.]', '', nickname) + + def get_id(self): + """ + Return the id (email) of the user. + """ + return self.email + + def set_password(self, password): + """ + Hash the password of the user. + """ + self.pwdhash = generate_password_hash(password) + + def check_password(self, password): + """ + Check the password of the user. + """ + return check_password_hash(self.pwdhash, password) + + def is_admin(self): + """ + Return True if the user has administrator rights. + """ + return len([role for role in self.roles if role.name == "admin"]) != 0 + + def __eq__(self, other): + return self.id == other.id + + def __repr__(self): + return '' % (self.nickname) + + +class Role(db.Model): + """ + Represent a role. + """ + id = db.Column(db.Integer, primary_key = True) + name = db.Column(db.String(), unique = True) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + + +class Feed(db.Model): + """ + Represent a station. + """ + id = db.Column(db.Integer, primary_key = True) + title = db.Column(db.String(), default="No title") + description = db.Column(db.String(), default="FR") + link = db.Column(db.String()) + site_link = db.Column(db.String(), default="") + email_notification = db.Column(db.Boolean(), default=False) + enabled = db.Column(db.Boolean(), default=True) + created_date = db.Column(db.DateTime(), default=datetime.now) + articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan', + order_by=desc("Article.date")) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + + def __repr__(self): + return '' % (self.title) + + def dump(self): + return {"id": self.id, + "title": self.title, + "description": self.description, + "link": self.link, + "site_link": self.site_link, + "nb_articles": self.articles.count(), + } + + +class Article(db.Model): + """ + Represent an article from a feed. + """ + id = db.Column(db.Integer, primary_key = True) + entry_id = db.Column(db.String()) + link = db.Column(db.String()) + title = db.Column(db.String()) + content = db.Column(db.String()) + readed = db.Column(db.Boolean(), default=False) + like = db.Column(db.Boolean(), default=False) + date = db.Column(db.DateTime(), default=datetime.now) + retrieved_date = db.Column(db.DateTime(), default=datetime.now) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) + + def previous_article(self): + """ + Returns the previous article (older). + """ + return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first() + + def next_article(self): + """ + Returns the next article (newer). + """ + return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first() + + def __repr__(self): + return json.dumps({ + "title": self.title, + "link": self.link, + "content": self.content + }) + def dump(self): + return {"id": self.id, + "title": self.title, + "link": self.link, + "content": self.content, + "readed": self.readed, + "like": self.like, + "date": self.date, + "retrieved_date": self.retrieved_date, + "feed_id": self.source.id, + "feed_name": self.source.title, + } diff --git a/pyaggr3g470r/rest.py b/pyaggr3g470r/rest.py deleted file mode 100644 index 6f5dd9b9..00000000 --- a/pyaggr3g470r/rest.py +++ /dev/null @@ -1,357 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2014/06/18 $" -__revision__ = "$Date: 2014/07/05 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "AGPLv3" - -import re -import dateutil.parser -from functools import wraps -from flask import g, Response, request, session, jsonify -from flask.ext.restful import Resource, reqparse - -import conf -if not conf.ON_HEROKU: - import pyaggr3g470r.search as fastsearch -from pyaggr3g470r import api, db -from pyaggr3g470r.models import User, Article, Feed - -def authenticate(func): - """ - Decorator for the authentication to the web services. - """ - @wraps(func) - def wrapper(*args, **kwargs): - if not getattr(func, 'authenticated', True): - return func(*args, **kwargs) - - # authentication based on the session (already logged on the site) - if 'email' in session or g.user.is_authenticated(): - return func(*args, **kwargs) - - # authentication via HTTP only - auth = request.authorization - try: - email = auth.username - user = User.query.filter(User.email == email).first() - if user and user.check_password(auth.password) and user.activation_key == "": - g.user = user - return func(*args, **kwargs) - except AttributeError: - pass - - return Response('', 401, - {'WWWAuthenticate':'Basic realm="Login Required"'}) - return wrapper - -class ArticleListAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, location = 'json') - self.reqparse.add_argument('content', type = unicode, location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('date', type = str, location = 'json') - self.reqparse.add_argument('feed_id', type = int, location = 'json') - super(ArticleListAPI, self).__init__() - - def get(self): - """ - Returns a list of articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) - - return jsonify(result= [{ - "id": article.id, - "title": article.title, - "link": article.link, - "content": article.content, - "readed": article.readed, - "like": article.like, - "date": article.date, - "retrieved_date": article.retrieved_date, - "feed_id": article.source.id, - "feed_name": article.source.title - } - for article in articles] - ) - - def post(self): - """ - POST method - Create a new article. - """ - args = self.reqparse.parse_args() - article_dict = {} - for k, v in args.iteritems(): - if v != None: - article_dict[k] = v - else: - return {"message":"Missing argument: %s." % (k,)} - article_date = None - try: - article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) - except: - try: # trying to clean date field from letters - article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) - except: - return jsonify({"message":"Bad format for the date."}) - article = Article(link=article_dict["link"], title=article_dict["title"], - content=article_dict["content"], readed=False, like=False, - date=article_date, user_id=g.user.id, - feed_id=article_dict["feed_id"]) - feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() - feed.articles.append(article) - try: - db.session.commit() - return jsonify({"message":"ok"}) - except: - return jsonify({"message":"Impossible to create the article."}) - -class ArticleAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('like', type = bool, location = 'json') - self.reqparse.add_argument('readed', type = bool, location = 'json') - super(ArticleAPI, self).__init__() - - def get(self, id=None): - """ - Returns an article. - """ - result = [] - if id is not None: - article = Article.query.filter(Article.user_id == g.user.id, Article.id == id).first() - if article is not None: - if not article.readed: - article.readed = True - db.session.commit() - result.append(article) - - return jsonify(result= [{ - "id": article.id, - "title": article.title, - "link": article.link, - "content": article.content, - "readed": article.readed, - "like": article.like, - "date": article.date, - "retrieved_date": article.retrieved_date, - "feed_id": article.source.id, - "feed_name": article.source.title - } - for article in result] - ) - - def put(self, id): - """ - Update an article. - It is only possible to update the status ('like' and 'readed') of an article. - """ - args = self.reqparse.parse_args() - article = Article.query.filter(Article.id == id).first() - if article is not None and article.source.subscriber.id == g.user.id: - if None is not args.get('like', None): - article.like = args['like'] - if None is not args.get('readed', None): - article.readed = args['readed'] - db.session.commit() - - try: - fastsearch.delete_article(g.user.id, article.feed_id, article.id) - except: - pass - - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Article not found.'}) - - def delete(self, id): - """ - Delete an article. - """ - article = Article.query.filter(Article.id == id).first() - if article is not None and article.source.subscriber.id == g.user.id: - db.session.delete(article) - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Article not found.'}) - -api.add_resource(ArticleListAPI, '/api/v1.0/articles', endpoint = 'articles.json') -api.add_resource(ArticleAPI, '/api/v1.0/articles/', endpoint = 'article.json') - -class FeedListAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('description', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('site_link', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('email_notification', type = bool, default = False, location = 'json') - self.reqparse.add_argument('enabled', type = bool, default = True ,location = 'json') - super(FeedListAPI, self).__init__() - - def get(self): - """ - Returns a list of feeds. - """ - return jsonify(result= [{ - "id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "email_notification": feed.email_notification, - "enabled": feed.enabled, - "created_date": feed.created_date - } - for feed in g.user.feeds] - ) - - def post(self): - """ - POST method - Create a new feed. - """ - args = self.reqparse.parse_args() - feed_dict = {} - for k, v in args.iteritems(): - if v != None: - feed_dict[k] = v - else: - return jsonify({'message': 'missing argument: %s' % (k,)}) - new_feed = Feed(title=feed_dict["title"], description=feed_dict["description"], - link=feed_dict["link"], site_link=feed_dict["site_link"], - email_notification=feed_dict["email_notification"], - enabled=feed_dict["enabled"]) - g.user.feeds.append(new_feed) - try: - db.session.commit() - return jsonify({"message":"ok"}) - except: - return jsonify({'message': 'Impossible to create the feed.'}) - -class FeedAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, location = 'json') - self.reqparse.add_argument('description', type = unicode, location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('site_link', type = unicode, location = 'json') - self.reqparse.add_argument('email_notification', type = bool, location = 'json') - self.reqparse.add_argument('enabled', type = bool ,location = 'json') - super(FeedAPI, self).__init__() - - def get(self, id=None): - """ - Returns a feed. - """ - result = [] - if id is not None: - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - result.append(feed) - return jsonify(result= [{ - "id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "nb_articles": feed.articles.count() - } - for feed in result] - ) - return jsonify({'message': 'Feed not found'}) - - def put(self, id): - """ - Update a feed. - """ - args = self.reqparse.parse_args() - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - if None is not args.get('title', None): - feed.title = args['title'] - if None is not args.get('description', None): - feed.description = args['description'] - if None is not args.get('link', None): - feed.link = args['link'] - if None is not args.get('site_link', None): - feed.site_link = args['site_link'] - if None is not args.get('email_notification', None): - feed.email_notification = args['email_notification'] - if None is not args.get('enabled', None): - feed.enabled = args['enabled'] - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Feed not found.'}) - - def delete(self, id): - """ - Delete a feed. - """ - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - db.session.delete(feed) - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Feed not found.'}) - -api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') -api.add_resource(FeedAPI, '/api/v1.0/feeds/', endpoint = 'feed.json') \ No newline at end of file diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views.py deleted file mode 100644 index f0422c3c..00000000 --- a/pyaggr3g470r/views.py +++ /dev/null @@ -1,930 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 5.3 $" -__date__ = "$Date: 2010/01/29 $" -__revision__ = "$Date: 2014/08/27 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "AGPLv3" - -import os -import datetime -from collections import namedtuple -from flask import abort, render_template, request, flash, session, \ - url_for, redirect, g, current_app, make_response, jsonify -from flask.ext.login import LoginManager, login_user, logout_user, \ - login_required, current_user, AnonymousUserMixin -from flask.ext.principal import Principal, Identity, AnonymousIdentity, \ - identity_changed, identity_loaded, Permission,\ - RoleNeed, UserNeed -from flask.ext.babel import gettext -from sqlalchemy import desc, func, or_ -from sqlalchemy.exc import IntegrityError -from werkzeug import generate_password_hash - -import conf -from pyaggr3g470r import utils, notifications, export, duplicate -from pyaggr3g470r import app, db, allowed_file, babel -from pyaggr3g470r.models import User, Feed, Article, Role -from pyaggr3g470r.decorators import feed_access_required -from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \ - ProfileForm, InformationMessageForm, RecoverPasswordForm -if not conf.ON_HEROKU: - import pyaggr3g470r.search as fastsearch - - -Principal(app) -# Create a permission with a single Need, in this case a RoleNeed. -admin_permission = Permission(RoleNeed('admin')) - -login_manager = LoginManager() -login_manager.init_app(app) - -# -# Management of the user's session. -# -@identity_loaded.connect_via(app) -def on_identity_loaded(sender, identity): - # Set the identity user object - identity.user = current_user - - # Add the UserNeed to the identity - if hasattr(current_user, 'id'): - identity.provides.add(UserNeed(current_user.id)) - - # Assuming the User model has a list of roles, update the - # identity with the roles that the user provides - if hasattr(current_user, 'roles'): - for role in current_user.roles: - identity.provides.add(RoleNeed(role.name)) - -@app.before_request -def before_request(): - g.user = current_user - if g.user.is_authenticated(): - g.user.last_seen = datetime.datetime.utcnow() - db.session.add(g.user) - db.session.commit() - -@login_manager.user_loader -def load_user(email): - # Return an instance of the User model - return User.query.filter(User.email == email).first() - - -# -# Custom error pages. -# -@app.errorhandler(401) -def authentication_required(e): - flash(gettext('Authentication required.'), 'info') - return redirect(url_for('login')) - -@app.errorhandler(403) -def authentication_failed(e): - flash(gettext('Forbidden.'), 'danger') - return redirect(url_for('home')) - -@app.errorhandler(404) -def page_not_found(e): - return render_template('errors/404.html'), 404 - -@app.errorhandler(500) -def internal_server_error(e): - return render_template('errors/500.html'), 500 - - -def redirect_url(default='home'): - return request.args.get('next') or \ - request.referrer or \ - url_for(default) - -@babel.localeselector -def get_locale(): - """ - Called before each request to give us a chance to choose - the language to use when producing its response. - """ - return request.accept_languages.best_match(conf.LANGUAGES.keys()) - -@babel.timezoneselector -def get_timezone(): - try: - return conf.TIME_ZONE[get_locale()] - except: - return conf.TIME_ZONE["en"] - -# -# Views. -# -@app.route('/login', methods=['GET', 'POST']) -def login(): - """ - Log in view. - """ - if g.user is not None and g.user.is_authenticated(): - return redirect(url_for('home')) - - g.user = AnonymousUserMixin() - form = SigninForm() - - if form.validate_on_submit(): - user = User.query.filter(User.email == form.email.data).first() - login_user(user) - g.user = user - session['email'] = form.email.data - identity_changed.send(current_app._get_current_object(), identity=Identity(user.id)) - flash(gettext("Logged in successfully."), 'success') - return redirect(url_for('home')) - return render_template('login.html', form=form) - -@app.route('/logout') -@login_required -def logout(): - """ - Log out view. Removes the user information from the session. - """ - session.pop('email', None) - - # Remove the user information from the session - logout_user() - - # Remove session keys set by Flask-Principal - for key in ('identity.name', 'identity.auth_type'): - session.pop(key, None) - - # Tell Flask-Principal the user is anonymous - identity_changed.send(current_app._get_current_object(), identity=AnonymousIdentity()) - - flash(gettext("Logged out successfully."), 'success') - return redirect(url_for('login')) - -@app.route('/signup', methods=['GET', 'POST']) -def signup(): - """ - Signup page. - """ - if int(os.environ.get("SELF_REGISTRATION", 0)) != 1: - flash(gettext("Self-registration is disabled."), 'warning') - return redirect(url_for('home')) - if g.user is not None and g.user.is_authenticated(): - return redirect(url_for('home')) - - form = SignupForm() - - if form.validate_on_submit(): - role_user = Role.query.filter(Role.name == "user").first() - user = User(nickname=form.nickname.data, - email=form.email.data, - pwdhash=generate_password_hash(form.password.data)) - user.roles = [role_user] - db.session.add(user) - try: - db.session.commit() - except IntegrityError: - flash(gettext('Email already used.'), 'warning') - return render_template('signup.html', form=form) - - # Send the confirmation email - try: - notifications.new_account_notification(user) - except Exception as e: - flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') - return redirect(url_for('home')) - - flash(gettext('Your account has been created. Check your mail to confirm it.'), 'success') - return redirect(url_for('home')) - - return render_template('signup.html', form=form) - -@app.route('/') -@login_required -def home(): - """ - Home page for connected users. Displays by default unread articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) - unread = db.session.query(Article.feed_id, func.count(Article.id))\ - .filter(Article.readed == False, Article.user_id == g.user.id)\ - .group_by(Article.feed_id).all() - def gen_url(filter_=filter_, limit=limit, feed=feed_id): - return '?filter_=%s&limit=%s&feed=%d' % (filter_, limit, feed) - return render_template('home.html', gen_url=gen_url, feed_id=feed_id, - filter_=filter_, limit=limit, feeds=feeds, - unread=dict(unread), articles=articles.all()) - - -@app.route('/article/redirect/', methods=['GET']) -@login_required -def redirect_to_article(article_id): - article = Article.query.filter(Article.id == article_id, - Article.user_id == g.user.id).first() - if article is None: - abort(404) - article.readed = True - db.session.commit() - return redirect(article.link) - - -@app.route('/fetch', methods=['GET']) -@app.route('/fetch/', methods=['GET']) -@login_required -def fetch(feed_id=None): - """ - Triggers the download of news. - News are downloaded in a separated process, mandatory for Heroku. - """ - utils.fetch(g.user.id, feed_id) - flash(gettext("Downloading articles..."), 'info') - return redirect(redirect_url()) - -@app.route('/about', methods=['GET']) -def about(): - """ - 'About' page. - """ - return render_template('about.html') - -@app.route('/feeds', methods=['GET']) -@login_required -def feeds(): - """ - Lists the subscribed feeds in a table. - """ - user = User.query.filter(User.email == g.user.email).first() - return render_template('feeds.html', feeds=user.feeds) - -@app.route('/feed/', methods=['GET']) -@login_required -@feed_access_required -def feed(feed_id=None): - """ - Presents detailed information about a feed. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - word_size = 6 - articles = feed.articles.all() - nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all()) - top_words = utils.top_words(articles, n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) - - today = datetime.datetime.now() - try: - last_article = articles[0].date - first_article = articles[-1].date - delta = last_article - first_article - average = round(float(len(articles)) / abs(delta.days), 2) - except: - last_article = datetime.datetime.fromtimestamp(0) - first_article = datetime.datetime.fromtimestamp(0) - delta = last_article - first_article - average = 0 - elapsed = today - last_article - - return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \ - first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \ - average=average, delta=delta, elapsed=elapsed) - -@app.route('/article/', methods=['GET']) -@login_required -def article(article_id=None): - """ - Presents the content of an article. - """ - article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() - if article is not None: - if not article.readed: - article.readed = True - db.session.commit() - - previous_article = article.previous_article() - if previous_article is None: - previous_article = article.source.articles[0] - next_article = article.next_article() - if next_article is None: - next_article = article.source.articles[-1] - - return render_template('article.html', head_title=utils.clear_string(article.title), - article=article, - previous_article=previous_article, next_article=next_article) - flash(gettext("This article do not exist."), 'warning') - return redirect(url_for('home')) - - -@app.route('/mark_as/', methods=['GET']) -@app.route('/mark_as//feed/', methods=['GET']) -@app.route('/mark_as//article/', methods=['GET']) -@login_required -@feed_access_required -def mark_as(new_value='read', feed_id=None, article_id=None): - """ - Mark all unreaded articles as read. - """ - readed = new_value == 'read' - articles = Article.query.filter(Article.user_id == g.user.id) - if feed_id is not None: - articles = articles.filter(Article.feed_id == feed_id) - message = 'Feed marked as %s.' - elif article_id is not None: - articles = articles.filter(Article.id == article_id) - message = 'Article marked as %s.' - else: - message = 'All article marked as %s.' - articles.filter(Article.readed == (not readed)).update({"readed": readed}) - flash(gettext(message % new_value), 'info') - db.session.commit() - if readed: - return redirect(redirect_url()) - return redirect(url_for('home')) - -@app.route('/like/', methods=['GET']) -@login_required -def like(article_id=None): - """ - Mark or unmark an article as favorites. - """ - Article.query.filter(Article.user_id == g.user.id, Article.id == article_id). \ - update({ - "like": not Article.query.filter(Article.id == article_id).first().like - }) - db.session.commit() - return redirect(redirect_url()) - -@app.route('/delete/', methods=['GET']) -@login_required -def delete(article_id=None): - """ - Delete an article from the database. - """ - article = Article.query.filter(Article.id == article_id).first() - if article is not None and article.source.subscriber.id == g.user.id: - db.session.delete(article) - db.session.commit() - try: - fastsearch.delete_article(g.user.id, article.feed_id, article.id) - except: - pass - flash(gettext('Article') + ' ' + article.title + ' ' + gettext('deleted.'), 'success') - return redirect(redirect_url()) - else: - flash(gettext('This article do not exist.'), 'danger') - return redirect(url_for('home')) - -@app.route('/articles/', methods=['GET']) -@app.route('/articles//', methods=['GET']) -@login_required -@feed_access_required -def articles(feed_id=None, nb_articles=-1): - """ - List articles of a feed. - The administrator of the platform is able to access to this view for every users. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - new_feed = feed - if len(feed.articles.all()) <= nb_articles: - nb_articles = -1 - if nb_articles == -1: - nb_articles = int(1e9) - new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \ - Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles) - return render_template('articles.html', feed=new_feed, nb_articles=nb_articles) - -@app.route('/favorites', methods=['GET']) -@login_required -def favorites(): - """ - List favorites articles. - """ - feeds_with_like = Feed.query.filter(Feed.user_id == g.user.id, Feed.articles.any(like=True)) - result, nb_favorites = [], 0 - light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) - for feed in feeds_with_like: - articles = Article.query.filter(Article.user_id == g.user.id, Article.feed_id == feed.id, Article.like == True).all() - result.append(light_feed(feed.id, feed.title, articles)) - nb_favorites += len(articles) - return render_template('favorites.html', feeds=result, nb_favorites=nb_favorites) - -@app.route('/unread/', methods=['GET']) -@app.route('/unread', methods=['GET']) -@login_required -def unread(feed_id=None): - """ - List unread articles. - """ - if feed_id is not None: - feeds_with_unread = Feed.query.filter(Feed.user_id == g.user.id, Feed.id == feed_id) - else: - feeds_with_unread = Feed.query.filter(Feed.user_id == g.user.id, Feed.articles.any(readed=False)) - result, nb_unread = [], 0 - light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) - for feed in feeds_with_unread: - articles = Article.query.filter(Article.user_id == g.user.id, Article.feed_id == feed.id, Article.readed == False).all() - result.append(light_feed(feed.id, feed.title, articles)) - nb_unread += len(articles) - return render_template('unread.html', feeds=result, nb_unread=nb_unread) - -@app.route('/inactives', methods=['GET']) -@login_required -def inactives(): - """ - List of inactive feeds. - """ - nb_days = int(request.args.get('nb_days', 365)) - user = User.query.filter(User.id == g.user.id).first() - today = datetime.datetime.now() - inactives = [] - for feed in user.feeds: - try: - last_post = feed.articles[0].date - except IndexError: - continue - elapsed = today - last_post - if elapsed > datetime.timedelta(days=nb_days): - inactives.append((feed, elapsed)) - return render_template('inactives.html', inactives=inactives, nb_days=nb_days) - -@app.route('/duplicates/', methods=['GET']) -@login_required -def duplicates(feed_id=None): - """ - Return duplicates article for a feed. - """ - feed = Feed.query.filter(Feed.user_id == g.user.id, Feed.id == feed_id).first() - duplicates = [] - duplicates = duplicate.compare_documents(feed) - return render_template('duplicates.html', duplicates=duplicates, feed=feed) - -@app.route('/index_database', methods=['GET']) -@login_required -def index_database(): - """ - Index all the database. - """ - if not conf.ON_HEROKU: - try: - fastsearch.create_index(g.user.id) - flash(gettext('Indexing database...'), 'success') - except Exception as e: - flash(gettext('An error occured') + ' (%s).' % e, 'danger') - return redirect(url_for('home')) - else: - flash(gettext('Option not available on Heroku.'), 'success') - return redirect(url_for('home')) - -@app.route('/export', methods=['GET']) -@login_required -def export_articles(): - """ - Export all articles to HTML or JSON. - """ - user = User.query.filter(User.id == g.user.id).first() - if request.args.get('format') == "HTML": - # Export to HTML - try: - archive_file, archive_file_name = export.export_html(user) - except: - flash(gettext("Error when exporting articles."), 'danger') - return redirect(redirect_url()) - response = make_response(archive_file) - response.headers['Content-Type'] = 'application/x-compressed' - response.headers['Content-Disposition'] = 'attachment; filename='+archive_file_name - elif request.args.get('format') == "JSON": - # Export to JSON - try: - json_result = export.export_json(user) - except: - flash(gettext("Error when exporting articles."), 'danger') - return redirect(redirect_url()) - response = make_response(json_result) - response.mimetype = 'application/json' - response.headers["Content-Disposition"] = 'attachment; filename=account.json' - else: - flash(gettext('Export format not supported.'), 'warning') - return redirect(redirect_url()) - return response - -@app.route('/export_opml', methods=['GET']) -@login_required -def export_opml(): - """ - Export all feeds to OPML. - """ - user = User.query.filter(User.id == g.user.id).first() - response = make_response(render_template('opml.xml', user=user, now=datetime.datetime.now())) - response.headers['Content-Type'] = 'application/xml' - response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml' - return response - -@app.route('/search', methods=['GET']) -@login_required -def search(): - """ - Search articles corresponding to the query. - """ - if conf.ON_HEROKU: - flash(gettext("Full text search is not yet implemented for Heroku."), "warning") - return redirect(url_for('home')) - user = User.query.filter(User.id == g.user.id).first() - - search_result, result = [], [] - nb_articles = 0 - - query = request.args.get('query', None) - if query is not None: - try: - search_result, nb_articles = fastsearch.search(user.id, query) - except Exception as e: - flash(gettext('An error occured') + ' (%s).' % e, 'danger') - light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) - for feed_id in search_result: - for feed in user.feeds: - if feed.id == feed_id: - articles = [] - for article_id in search_result[feed_id]: - current_article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() - articles.append(current_article) - articles = sorted(articles, key=lambda t: t.date, reverse=True) - result.append(light_feed(feed.id, feed.title, articles)) - break - return render_template('search.html', feeds=result, nb_articles=nb_articles, query=query) - -@app.route('/management', methods=['GET', 'POST']) -@login_required -def management(): - """ - Display the management page. - """ - if request.method == 'POST': - if None != request.files.get('opmlfile', None): - # Import an OPML file - data = request.files.get('opmlfile', None) - if not allowed_file(data.filename): - flash(gettext('File not allowed.'), 'danger') - else: - try: - nb = utils.import_opml(g.user.email, data.read()) - utils.fetch(g.user.email, None) - flash(str(nb) + ' ' + gettext('feeds imported.'), "success") - flash(gettext("Downloading articles..."), 'info') - except: - flash(gettext("Impossible to import the new feeds."), "danger") - elif None != request.files.get('jsonfile', None): - # Import an account - data = request.files.get('jsonfile', None) - if not allowed_file(data.filename): - flash(gettext('File not allowed.'), 'danger') - else: - try: - nb = utils.import_json(g.user.email, data.read()) - flash(gettext('Account imported.'), "success") - except: - flash(gettext("Impossible to import the account."), "danger") - else: - flash(gettext('File not allowed.'), 'danger') - - form = AddFeedForm() - nb_feeds = len(g.user.feeds.all()) - articles = Article.query.filter(Article.user_id == g.user.id) - nb_articles = articles.count() - nb_unread_articles = articles.filter(Article.readed == False).count() - return render_template('management.html', user=g.user, form=form, - nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, - not_on_heroku = not conf.ON_HEROKU) - -@app.route('/history', methods=['GET']) -@login_required -def history(): - user = User.query.filter(User.id == g.user.id).first() - return render_template('history.html') - -@app.route('/bookmarklet', methods=['GET']) -@app.route('/create_feed', methods=['GET', 'POST']) -@app.route('/edit_feed/', methods=['GET', 'POST']) -@login_required -@feed_access_required -def edit_feed(feed_id=None): - """ - Add or edit a feed. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - form = AddFeedForm() - - if request.method == 'POST': - if form.validate() == False: - return render_template('edit_feed.html', form=form) - if feed_id is not None: - # Edit an existing feed - form.populate_obj(feed) - db.session.commit() - flash(gettext('Feed successfully updated.'), 'success') - return redirect('/edit_feed/' + str(feed_id)) - else: - # Create a new feed - existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data] - if len(existing_feed) == 0: - new_feed = Feed(title=form.title.data, description="", link=form.link.data, \ - site_link=form.site_link.data, email_notification=form.email_notification.data, \ - enabled=form.enabled.data) - g.user.feeds.append(new_feed) - #user.feeds = sorted(user.feeds, key=lambda t: t.title.lower()) - db.session.commit() - flash(gettext('Feed successfully created.'), 'success') - - utils.fetch(g.user.id, Feed.query.filter(Feed.link == form.link.data).first().id) - flash(gettext("Downloading articles for the new feed..."), 'info') - - return redirect('/edit_feed/' + str(new_feed.id)) - else: - flash(gettext('Feed already in the database.'), 'warning') - return redirect('/edit_feed/' + str(existing_feed[0].id)) - - if request.method == 'GET': - if feed_id is not None: - form = AddFeedForm(obj=feed) - return render_template('edit_feed.html', action=gettext("Edit the feed"), form=form, feed=feed, \ - not_on_heroku = not conf.ON_HEROKU) - - # Enable the user to add a feed with a bookmarklet - if None is not request.args.get('url', None): - existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)] - if len(existing_feed) == 0: - g.user.feeds.append(Feed(link=request.args.get('url', None))) - db.session.commit() - return jsonify({"message":"ok"}) - return jsonify({"message":"Feed already in the database."}) - - # Return an empty form in order to create a new feed - return render_template('edit_feed.html', action=gettext("Add a feed"), form=form, \ - not_on_heroku = not conf.ON_HEROKU) - -@app.route('/delete_feed/', methods=['GET']) -@login_required -@feed_access_required -def delete_feed(feed_id=None): - """ - Delete a feed with all associated articles. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - db.session.delete(feed) - db.session.commit() - flash(gettext('Feed') + ' ' + feed.title + ' ' + gettext('successfully deleted.'), 'success') - return redirect(redirect_url()) - -@app.route('/profile', methods=['GET', 'POST']) -@login_required -def profile(): - """ - Edit the profile of the currently logged user. - """ - user = User.query.filter(User.email == g.user.email).first() - form = ProfileForm() - - if request.method == 'POST': - if form.validate(): - form.populate_obj(user) - if form.password.data != "": - user.set_password(form.password.data) - db.session.commit() - flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') - return redirect(url_for('profile')) - else: - return render_template('profile.html', form=form) - - if request.method == 'GET': - form = ProfileForm(obj=user) - return render_template('profile.html', user=user, form=form) - -@app.route('/delete_account', methods=['GET']) -@login_required -def delete_account(): - """ - Delete the account of the user (with all its data). - """ - user = User.query.filter(User.email == g.user.email).first() - if user is not None: - db.session.delete(user) - db.session.commit() - flash(gettext('Your account has been deleted.'), 'success') - else: - flash(gettext('This user does not exist.'), 'danger') - return redirect(url_for('login')) - -@app.route('/expire_articles', methods=['GET']) -@login_required -def expire_articles(): - """ - Delete articles older than the given number of weeks. - """ - current_time = datetime.datetime.utcnow() - weeks_ago = current_time - datetime.timedelta(weeks=int(request.args.get('weeks', 10))) - articles_to_delete = Article.query.filter(User.email == g.user.email, or_(Article.date < weeks_ago, Article.retrieved_date < weeks_ago)) - for article in articles_to_delete: - db.session.delete(article) - flash(gettext('Articles deleted.'), 'info') - db.session.commit() - return redirect(redirect_url()) - -@app.route('/confirm_account/', methods=['GET']) -def confirm_account(activation_key=None): - """ - Confirm the account of a user. - """ - if activation_key != "": - user = User.query.filter(User.activation_key == activation_key).first() - if user is not None: - user.activation_key = "" - db.session.commit() - flash(gettext('Your account has been confirmed.'), 'success') - else: - flash(gettext('Impossible to confirm this account.'), 'danger') - return redirect(url_for('login')) - -@app.route('/recover', methods=['GET', 'POST']) -def recover(): - """ - Enables the user to recover its account when he has forgotten - its password. - """ - import string - import random - form = RecoverPasswordForm() - - if request.method == 'POST': - if form.validate(): - user = User.query.filter(User.email == form.email.data).first() - characters = string.ascii_letters + string.digits - password = "".join(random.choice(characters) for x in range(random.randint(8, 16))) - user.set_password(password) - db.session.commit() - - # Send the confirmation email - try: - notifications.new_password_notification(user, password) - flash(gettext('New password sent to your address.'), 'success') - except Exception as e: - flash(gettext('Problem while sending your new password.') + ': ' + str(e), 'danger') - - return redirect(url_for('login')) - return render_template('recover.html', form=form) - - if request.method == 'GET': - return render_template('recover.html', form=form) - -# -# Views dedicated to administration tasks. -# -@app.route('/admin/dashboard', methods=['GET', 'POST']) -@login_required -@admin_permission.require(http_exception=403) -def dashboard(): - """ - Adminstrator's dashboard. - """ - form = InformationMessageForm() - - if request.method == 'POST': - if form.validate(): - try: - notifications.information_message(form.subject.data, form.message.data) - except Exception as e: - flash(gettext('Problem while sending email') + ': ' + str(e), 'danger') - - users = User.query.all() - return render_template('admin/dashboard.html', users=users, current_user=g.user, form=form) - -@app.route('/admin/create_user', methods=['GET', 'POST']) -@app.route('/admin/edit_user/', methods=['GET', 'POST']) -@login_required -@admin_permission.require(http_exception=403) -def create_user(user_id=None): - """ - Create or edit a user. - """ - form = ProfileForm() - - if request.method == 'POST': - if form.validate(): - role_user = Role.query.filter(Role.name == "user").first() - if user_id is not None: - # Edit a user - user = User.query.filter(User.id == user_id).first() - form.populate_obj(user) - if form.password.data != "": - user.set_password(form.password.data) - db.session.commit() - flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') - else: - # Create a new user - user = User(nickname=form.nickname.data, - email=form.email.data, - pwdhash=generate_password_hash(form.password.data)) - user.roles.extend([role_user]) - user.activation_key = "" - db.session.add(user) - db.session.commit() - flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully created.'), 'success') - return redirect("/admin/edit_user/"+str(user.id)) - else: - return redirect(url_for('create_user')) - - if request.method == 'GET': - if user_id is not None: - user = User.query.filter(User.id == user_id).first() - form = ProfileForm(obj=user) - message = gettext('Edit the user') + ' ' + user.nickname + '' - else: - form = ProfileForm() - message = gettext('Add a new user') - return render_template('/admin/create_user.html', form=form, message=message) - -@app.route('/admin/user/', methods=['GET']) -@login_required -@admin_permission.require(http_exception=403) -def user(user_id=None): - """ - See information about a user (stations, etc.). - """ - user = User.query.filter(User.id == user_id).first() - if user is not None: - return render_template('/admin/user.html', user=user) - else: - flash(gettext('This user does not exist.'), 'danger') - return redirect(redirect_url()) - -@app.route('/admin/delete_user/', methods=['GET']) -@login_required -@admin_permission.require(http_exception=403) -def delete_user(user_id=None): - """ - Delete a user (with all its data). - """ - user = User.query.filter(User.id == user_id).first() - if user is not None: - db.session.delete(user) - db.session.commit() - flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully deleted.'), 'success') - else: - flash(gettext('This user does not exist.'), 'danger') - return redirect(redirect_url()) - -@app.route('/admin/enable_user/', methods=['GET']) -@app.route('/admin/disable_user/', methods=['GET']) -@login_required -@admin_permission.require() -def disable_user(user_id=None): - """ - Enable or disable the account of a user. - """ - user = User.query.filter(User.id == user_id).first() - if user is not None: - if user.activation_key != "": - - # Send the confirmation email - try: - notifications.new_account_activation(user) - user.activation_key = "" - flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully activated.'), 'success') - except Exception as e: - flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') - - else: - import random, hashlib - user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86] - flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success') - db.session.commit() - else: - flash(gettext('This user does not exist.'), 'danger') - return redirect(redirect_url()) diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py new file mode 100644 index 00000000..ad71048a --- /dev/null +++ b/pyaggr3g470r/views/__init__.py @@ -0,0 +1,2 @@ +from .views import * +from .api import * diff --git a/pyaggr3g470r/views/api.py b/pyaggr3g470r/views/api.py new file mode 100644 index 00000000..c220d0bc --- /dev/null +++ b/pyaggr3g470r/views/api.py @@ -0,0 +1,356 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.2 $" +__date__ = "$Date: 2014/06/18 $" +__revision__ = "$Date: 2014/07/05 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "AGPLv3" + +import re +import dateutil.parser +from functools import wraps +from flask import g, Response, request, session, jsonify +from flask.ext.restful import Resource, reqparse + +import conf +if not conf.ON_HEROKU: + import pyaggr3g470r.search as fastsearch +from pyaggr3g470r import api, db +from pyaggr3g470r.models import User, Article, Feed + + +def authenticate(func): + """ + Decorator for the authentication to the web services. + """ + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + # authentication based on the session (already logged on the site) + if 'email' in session or g.user.is_authenticated(): + return func(*args, **kwargs) + + # authentication via HTTP only + auth = request.authorization + try: + email = auth.username + user = User.query.filter(User.email == email).first() + if user and user.check_password(auth.password) and user.activation_key == "": + g.user = user + return func(*args, **kwargs) + except AttributeError: + pass + + return Response('', 401, + {'WWWAuthenticate':'Basic realm="Login Required"'}) + return wrapper + + +def to_response(func): + def wrapper(*args, **kwargs): + res = func(*args, **kwargs) + if type(res) is tuple and len(res) == 2 and type(res[1]) is int: + response = jsonify(**res[0]) + response.status_code = res[1] + if isinstance(res, Response): + return res + else: + response = jsonify(**res) + return response + return wrapper + + +class ArticleListAPI(Resource): + """ + Defines a RESTful API for Article elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', type=unicode, location='json') + self.reqparse.add_argument('content', type=unicode, location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('date', type=str, location='json') + self.reqparse.add_argument('feed_id', type=int, location='json') + super(ArticleListAPI, self).__init__() + + def get(self): + """ + Returns a list of articles. + """ + feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} + articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), + Article.user_id == g.user.id) + filter_ = request.args.get('filter_', 'unread') + feed_id = int(request.args.get('feed', 0)) + limit = request.args.get('limit', 1000) + if filter_ != 'all': + articles = articles.filter(Article.readed == (filter_ == 'read')) + if feed_id: + articles = articles.filter(Article.feed_id == feed_id) + + articles = articles.order_by(Article.date.desc()) + if limit != 'all': + limit = int(limit) + articles = articles.limit(limit) + + return {'result': [article.dump() for article in articles]} + + def post(self): + """ + POST method - Create a new article. + """ + args = self.reqparse.parse_args() + article_dict = {} + for k, v in args.iteritems(): + if v != None: + article_dict[k] = v + else: + return {"message": "Missing argument: %s." % (k,)}, 400 + article_date = None + try: + article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) + except: + try: # trying to clean date field from letters + article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) + except: + return jsonify({"message": "Bad format for the date."}), 400 + article = Article(link=article_dict["link"], title=article_dict["title"], + content=article_dict["content"], readed=False, like=False, + date=article_date, user_id=g.user.id, + feed_id=article_dict["feed_id"]) + feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() + feed.articles.append(article) + try: + db.session.commit() + return {"message": "ok"}, 201 + except: + return {"message": "Impossible to create the article."}, 500 + + +class ArticleAPI(Resource): + """ + Defines a RESTful API for Article elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('like', type=bool, location='json') + self.reqparse.add_argument('readed', type=bool, location= 'json') + super(ArticleAPI, self).__init__() + + def get_article_or_raise(self, article_id=None): + if article_id is None: + raise Exception({'message': 'Bad id'}, 400) + article = Article.query.filter(Article.id == article_id).first() + if article.source.subscriber.id != g.user.id: + return {'message': "Bad user for article."}, 403 + if article is None: + return {'message': 'Article not found'}, 404 + return article + + def get(self, id=None): + "Returns an article." + try: + article = self.get_article_or_raise(id) + except Exception, error: + return error.args + if not article.readed: + article.readed = True + db.session.commit() + return {'result': [article.dump()]} + + def put(self, id): + """ Update an article. It is only possible to update the status + ('like' and 'readed') of an article.""" + args = self.reqparse.parse_args() + try: + article = self.get_article_or_raise(id) + except Exception, error: + return error.args + if None is not args.get('like', None): + article.like = args['like'] + if None is not args.get('readed', None): + article.readed = args['readed'] + db.session.commit() + + try: + fastsearch.delete_article(g.user.id, article.feed_id, article.id) + except: + pass + return {"message": "ok"} + + def delete(self, id): + """ + Delete an article. + """ + try: + article = self.get_article_or_raise(id) + except Exception, error: + return error.args + db.session.delete(article) + db.session.commit() + return {"message": "ok"}, 204 + + +class FeedListAPI(Resource): + """ + Defines a RESTful API for Feed elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', + type=unicode, default="", location='json') + self.reqparse.add_argument('description', + type=unicode, default="", location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('site_link', + type=unicode, default="", location='json') + self.reqparse.add_argument('email_notification', + type=bool, default=False, location='json') + self.reqparse.add_argument('enabled', + type=bool, default=True, location='json') + super(FeedListAPI, self).__init__() + + def get(self): + """ + Returns a list of feeds. + """ + return {'result': [{"id": feed.id, + "title": feed.title, + "description": feed.description, + "link": feed.link, + "site_link": feed.site_link, + "email_notification": feed.email_notification, + "enabled": feed.enabled, + "created_date": feed.created_date, + } for feed in g.user.feeds]} + + def post(self): + """ + POST method - Create a new feed. + """ + args = self.reqparse.parse_args() + feed_dict = {} + for k, v in args.iteritems(): + if v != None: + feed_dict[k] = v + else: + return {'message': 'missing argument: %s' % (k,)}, 400 + new_feed = Feed(title=feed_dict["title"], + description=feed_dict["description"], + link=feed_dict["link"], + site_link=feed_dict["site_link"], + email_notification=feed_dict["email_notification"], + enabled=feed_dict["enabled"]) + g.user.feeds.append(new_feed) + try: + db.session.commit() + return {"message": "ok"} + except: + return {'message': 'Impossible to create the feed.'}, 500 + + +class FeedAPI(Resource): + """ + Defines a RESTful API for Feed elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', type=unicode, location='json') + self.reqparse.add_argument('description', + type=unicode, location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('site_link', type=unicode, location='json') + self.reqparse.add_argument('email_notification', + type=bool, location='json') + self.reqparse.add_argument('enabled', type=bool ,location='json') + super(FeedAPI, self).__init__() + + def get_feed_or_raise(self, feed_id=None): + if feed_id is None: + raise Exception({'message': 'Bad id'}, 400) + feed = Article.query.filter(Article.id == feed_id).first() + if feed.source.subscriber.id != g.user.id: + return {'message': "Bad user for article."}, 403 + if feed is None: + return {'message': 'Article not found'}, 404 + return feed + + def get(self, id=None): + "Returns a feed" + try: + feed = self.get_feed_or_raise(id) + except Exception, error: + return error.args + return {'result': [feed.dump()]} + + def put(self, id): + "Update a feed" + args = self.reqparse.parse_args() + try: + feed = self.get_feed_or_raise(id) + except Exception, error: + return error.args + if 'title' in args: + feed.title = args['title'] + if 'description' in args: + feed.description = args['description'] + if 'link' in args: + feed.link = args['link'] + if 'site_link' in args: + feed.site_link = args['site_link'] + if 'email_notification' in args: + feed.email_notification = args['email_notification'] + if 'enabled' in args: + feed.enabled = args['enabled'] + db.session.commit() + return {"message": "ok"} + + def delete(self, id): + """ + Delete a feed. + """ + try: + feed = self.get_feed_or_raise(id) + except Exception, error: + return error.args + db.session.delete(feed) + db.session.commit() + return {"message": "ok"}, 204 + + +api.add_resource(ArticleListAPI, '/api/v1.0/articles', + endpoint='articles.json') +api.add_resource(ArticleAPI, '/api/v1.0/articles/', + endpoint='article.json') +api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') +api.add_resource(FeedAPI, '/api/v1.0/feeds/', endpoint = 'feed.json') diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py new file mode 100644 index 00000000..0a50d8d0 --- /dev/null +++ b/pyaggr3g470r/views/views.py @@ -0,0 +1,956 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 5.3 $" +__date__ = "$Date: 2010/01/29 $" +__revision__ = "$Date: 2014/08/27 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "AGPLv3" + +import os +import json +import datetime +from collections import namedtuple +from flask import abort, render_template, request, flash, session, Response, \ + url_for, redirect, g, current_app, make_response, jsonify +from flask.ext.login import LoginManager, login_user, logout_user, \ + login_required, current_user, AnonymousUserMixin +from flask.ext.principal import Principal, Identity, AnonymousIdentity, \ + identity_changed, identity_loaded, Permission,\ + RoleNeed, UserNeed +from flask.ext.babel import gettext +from sqlalchemy import desc, func, or_ +from sqlalchemy.exc import IntegrityError +from werkzeug import generate_password_hash + +import conf +from pyaggr3g470r import utils, notifications, export, duplicate +from pyaggr3g470r import app, db, allowed_file, babel +from pyaggr3g470r.models import User, Feed, Article, Role +from pyaggr3g470r.decorators import feed_access_required +from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \ + ProfileForm, InformationMessageForm, RecoverPasswordForm +if not conf.ON_HEROKU: + import pyaggr3g470r.search as fastsearch + + +Principal(app) +# Create a permission with a single Need, in this case a RoleNeed. +admin_permission = Permission(RoleNeed('admin')) + +login_manager = LoginManager() +login_manager.init_app(app) + +# +# Management of the user's session. +# +@identity_loaded.connect_via(app) +def on_identity_loaded(sender, identity): + # Set the identity user object + identity.user = current_user + + # Add the UserNeed to the identity + if hasattr(current_user, 'id'): + identity.provides.add(UserNeed(current_user.id)) + + # Assuming the User model has a list of roles, update the + # identity with the roles that the user provides + if hasattr(current_user, 'roles'): + for role in current_user.roles: + identity.provides.add(RoleNeed(role.name)) + +@app.before_request +def before_request(): + g.user = current_user + if g.user.is_authenticated(): + g.user.last_seen = datetime.datetime.utcnow() + db.session.add(g.user) + db.session.commit() + +@login_manager.user_loader +def load_user(email): + # Return an instance of the User model + return User.query.filter(User.email == email).first() + + +# +# Custom error pages. +# +@app.errorhandler(401) +def authentication_required(e): + flash(gettext('Authentication required.'), 'info') + return redirect(url_for('login')) + +@app.errorhandler(403) +def authentication_failed(e): + flash(gettext('Forbidden.'), 'danger') + return redirect(url_for('home')) + +@app.errorhandler(404) +def page_not_found(e): + return render_template('errors/404.html'), 404 + +@app.errorhandler(500) +def internal_server_error(e): + return render_template('errors/500.html'), 500 + + +def redirect_url(default='home'): + return request.args.get('next') or \ + request.referrer or \ + url_for(default) + +@babel.localeselector +def get_locale(): + """ + Called before each request to give us a chance to choose + the language to use when producing its response. + """ + return request.accept_languages.best_match(conf.LANGUAGES.keys()) + +@babel.timezoneselector +def get_timezone(): + try: + return conf.TIME_ZONE[get_locale()] + except: + return conf.TIME_ZONE["en"] + +# +# Views. +# +@app.route('/login', methods=['GET', 'POST']) +def login(): + """ + Log in view. + """ + if g.user is not None and g.user.is_authenticated(): + return redirect(url_for('home')) + + g.user = AnonymousUserMixin() + form = SigninForm() + + if form.validate_on_submit(): + user = User.query.filter(User.email == form.email.data).first() + login_user(user) + g.user = user + session['email'] = form.email.data + identity_changed.send(current_app._get_current_object(), identity=Identity(user.id)) + flash(gettext("Logged in successfully."), 'success') + return redirect(url_for('home')) + return render_template('login.html', form=form) + +@app.route('/api/csrf', methods=['GET']) +def get_csrf(): + try: + data = json.loads(request.data) + except ValueError: + return Response(status=400) + email = data.get('email') + password = data.get('password') + if login is None or password is None: + return Response(status=401) + user = User.query.filter(User.email == email).first() + if not user: + return Reponse(status=404) + if not user.check_password(password): + return Reponse(status=401) + if not user.activation_key == "": + return Reponse(status=403) + login_user(user) + g.user = user + session['email'] = email + identity_changed.send(current_app._get_current_object(), + identity=Identity(user.id)) + return 'ok', 200 + + +@app.route('/logout') +@login_required +def logout(): + """ + Log out view. Removes the user information from the session. + """ + session.pop('email', None) + + # Remove the user information from the session + logout_user() + + # Remove session keys set by Flask-Principal + for key in ('identity.name', 'identity.auth_type'): + session.pop(key, None) + + # Tell Flask-Principal the user is anonymous + identity_changed.send(current_app._get_current_object(), identity=AnonymousIdentity()) + + flash(gettext("Logged out successfully."), 'success') + return redirect(url_for('login')) + +@app.route('/signup', methods=['GET', 'POST']) +def signup(): + """ + Signup page. + """ + if int(os.environ.get("SELF_REGISTRATION", 0)) != 1: + flash(gettext("Self-registration is disabled."), 'warning') + return redirect(url_for('home')) + if g.user is not None and g.user.is_authenticated(): + return redirect(url_for('home')) + + form = SignupForm() + + if form.validate_on_submit(): + role_user = Role.query.filter(Role.name == "user").first() + user = User(nickname=form.nickname.data, + email=form.email.data, + pwdhash=generate_password_hash(form.password.data)) + user.roles = [role_user] + db.session.add(user) + try: + db.session.commit() + except IntegrityError: + flash(gettext('Email already used.'), 'warning') + return render_template('signup.html', form=form) + + # Send the confirmation email + try: + notifications.new_account_notification(user) + except Exception as e: + flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') + return redirect(url_for('home')) + + flash(gettext('Your account has been created. Check your mail to confirm it.'), 'success') + return redirect(url_for('home')) + + return render_template('signup.html', form=form) + +@app.route('/') +@login_required +def home(): + """ + Home page for connected users. Displays by default unread articles. + """ + feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} + articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), + Article.user_id == g.user.id) + filter_ = request.args.get('filter_', 'unread') + feed_id = int(request.args.get('feed', 0)) + limit = request.args.get('limit', 1000) + if filter_ != 'all': + articles = articles.filter(Article.readed == (filter_ == 'read')) + if feed_id: + articles = articles.filter(Article.feed_id == feed_id) + + articles = articles.order_by(Article.date.desc()) + if limit != 'all': + limit = int(limit) + articles = articles.limit(limit) + unread = db.session.query(Article.feed_id, func.count(Article.id))\ + .filter(Article.readed == False, Article.user_id == g.user.id)\ + .group_by(Article.feed_id).all() + def gen_url(filter_=filter_, limit=limit, feed=feed_id): + return '?filter_=%s&limit=%s&feed=%d' % (filter_, limit, feed) + return render_template('home.html', gen_url=gen_url, feed_id=feed_id, + filter_=filter_, limit=limit, feeds=feeds, + unread=dict(unread), articles=articles.all()) + + +@app.route('/article/redirect/', methods=['GET']) +@login_required +def redirect_to_article(article_id): + article = Article.query.filter(Article.id == article_id, + Article.user_id == g.user.id).first() + if article is None: + abort(404) + article.readed = True + db.session.commit() + return redirect(article.link) + + +@app.route('/fetch', methods=['GET']) +@app.route('/fetch/', methods=['GET']) +@login_required +def fetch(feed_id=None): + """ + Triggers the download of news. + News are downloaded in a separated process, mandatory for Heroku. + """ + utils.fetch(g.user.id, feed_id) + flash(gettext("Downloading articles..."), 'info') + return redirect(redirect_url()) + +@app.route('/about', methods=['GET']) +def about(): + """ + 'About' page. + """ + return render_template('about.html') + +@app.route('/feeds', methods=['GET']) +@login_required +def feeds(): + """ + Lists the subscribed feeds in a table. + """ + user = User.query.filter(User.email == g.user.email).first() + return render_template('feeds.html', feeds=user.feeds) + +@app.route('/feed/', methods=['GET']) +@login_required +@feed_access_required +def feed(feed_id=None): + """ + Presents detailed information about a feed. + """ + feed = Feed.query.filter(Feed.id == feed_id).first() + word_size = 6 + articles = feed.articles.all() + nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all()) + top_words = utils.top_words(articles, n=50, size=int(word_size)) + tag_cloud = utils.tag_cloud(top_words) + + today = datetime.datetime.now() + try: + last_article = articles[0].date + first_article = articles[-1].date + delta = last_article - first_article + average = round(float(len(articles)) / abs(delta.days), 2) + except: + last_article = datetime.datetime.fromtimestamp(0) + first_article = datetime.datetime.fromtimestamp(0) + delta = last_article - first_article + average = 0 + elapsed = today - last_article + + return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \ + first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \ + average=average, delta=delta, elapsed=elapsed) + +@app.route('/article/', methods=['GET']) +@login_required +def article(article_id=None): + """ + Presents the content of an article. + """ + article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() + if article is not None: + if not article.readed: + article.readed = True + db.session.commit() + + previous_article = article.previous_article() + if previous_article is None: + previous_article = article.source.articles[0] + next_article = article.next_article() + if next_article is None: + next_article = article.source.articles[-1] + + return render_template('article.html', head_title=utils.clear_string(article.title), + article=article, + previous_article=previous_article, next_article=next_article) + flash(gettext("This article do not exist."), 'warning') + return redirect(url_for('home')) + + +@app.route('/mark_as/', methods=['GET']) +@app.route('/mark_as//feed/', methods=['GET']) +@app.route('/mark_as//article/', methods=['GET']) +@login_required +@feed_access_required +def mark_as(new_value='read', feed_id=None, article_id=None): + """ + Mark all unreaded articles as read. + """ + readed = new_value == 'read' + articles = Article.query.filter(Article.user_id == g.user.id) + if feed_id is not None: + articles = articles.filter(Article.feed_id == feed_id) + message = 'Feed marked as %s.' + elif article_id is not None: + articles = articles.filter(Article.id == article_id) + message = 'Article marked as %s.' + else: + message = 'All article marked as %s.' + articles.filter(Article.readed == (not readed)).update({"readed": readed}) + flash(gettext(message % new_value), 'info') + db.session.commit() + if readed: + return redirect(redirect_url()) + return redirect(url_for('home')) + +@app.route('/like/', methods=['GET']) +@login_required +def like(article_id=None): + """ + Mark or unmark an article as favorites. + """ + Article.query.filter(Article.user_id == g.user.id, Article.id == article_id). \ + update({ + "like": not Article.query.filter(Article.id == article_id).first().like + }) + db.session.commit() + return redirect(redirect_url()) + +@app.route('/delete/', methods=['GET']) +@login_required +def delete(article_id=None): + """ + Delete an article from the database. + """ + article = Article.query.filter(Article.id == article_id).first() + if article is not None and article.source.subscriber.id == g.user.id: + db.session.delete(article) + db.session.commit() + try: + fastsearch.delete_article(g.user.id, article.feed_id, article.id) + except: + pass + flash(gettext('Article') + ' ' + article.title + ' ' + gettext('deleted.'), 'success') + return redirect(redirect_url()) + else: + flash(gettext('This article do not exist.'), 'danger') + return redirect(url_for('home')) + +@app.route('/articles/', methods=['GET']) +@app.route('/articles//', methods=['GET']) +@login_required +@feed_access_required +def articles(feed_id=None, nb_articles=-1): + """ + List articles of a feed. + The administrator of the platform is able to access to this view for every users. + """ + feed = Feed.query.filter(Feed.id == feed_id).first() + new_feed = feed + if len(feed.articles.all()) <= nb_articles: + nb_articles = -1 + if nb_articles == -1: + nb_articles = int(1e9) + new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \ + Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles) + return render_template('articles.html', feed=new_feed, nb_articles=nb_articles) + +@app.route('/favorites', methods=['GET']) +@login_required +def favorites(): + """ + List favorites articles. + """ + feeds_with_like = Feed.query.filter(Feed.user_id == g.user.id, Feed.articles.any(like=True)) + result, nb_favorites = [], 0 + light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) + for feed in feeds_with_like: + articles = Article.query.filter(Article.user_id == g.user.id, Article.feed_id == feed.id, Article.like == True).all() + result.append(light_feed(feed.id, feed.title, articles)) + nb_favorites += len(articles) + return render_template('favorites.html', feeds=result, nb_favorites=nb_favorites) + +@app.route('/unread/', methods=['GET']) +@app.route('/unread', methods=['GET']) +@login_required +def unread(feed_id=None): + """ + List unread articles. + """ + if feed_id is not None: + feeds_with_unread = Feed.query.filter(Feed.user_id == g.user.id, Feed.id == feed_id) + else: + feeds_with_unread = Feed.query.filter(Feed.user_id == g.user.id, Feed.articles.any(readed=False)) + result, nb_unread = [], 0 + light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) + for feed in feeds_with_unread: + articles = Article.query.filter(Article.user_id == g.user.id, Article.feed_id == feed.id, Article.readed == False).all() + result.append(light_feed(feed.id, feed.title, articles)) + nb_unread += len(articles) + return render_template('unread.html', feeds=result, nb_unread=nb_unread) + +@app.route('/inactives', methods=['GET']) +@login_required +def inactives(): + """ + List of inactive feeds. + """ + nb_days = int(request.args.get('nb_days', 365)) + user = User.query.filter(User.id == g.user.id).first() + today = datetime.datetime.now() + inactives = [] + for feed in user.feeds: + try: + last_post = feed.articles[0].date + except IndexError: + continue + elapsed = today - last_post + if elapsed > datetime.timedelta(days=nb_days): + inactives.append((feed, elapsed)) + return render_template('inactives.html', inactives=inactives, nb_days=nb_days) + +@app.route('/duplicates/', methods=['GET']) +@login_required +def duplicates(feed_id=None): + """ + Return duplicates article for a feed. + """ + feed = Feed.query.filter(Feed.user_id == g.user.id, Feed.id == feed_id).first() + duplicates = [] + duplicates = duplicate.compare_documents(feed) + return render_template('duplicates.html', duplicates=duplicates, feed=feed) + +@app.route('/index_database', methods=['GET']) +@login_required +def index_database(): + """ + Index all the database. + """ + if not conf.ON_HEROKU: + try: + fastsearch.create_index(g.user.id) + flash(gettext('Indexing database...'), 'success') + except Exception as e: + flash(gettext('An error occured') + ' (%s).' % e, 'danger') + return redirect(url_for('home')) + else: + flash(gettext('Option not available on Heroku.'), 'success') + return redirect(url_for('home')) + +@app.route('/export', methods=['GET']) +@login_required +def export_articles(): + """ + Export all articles to HTML or JSON. + """ + user = User.query.filter(User.id == g.user.id).first() + if request.args.get('format') == "HTML": + # Export to HTML + try: + archive_file, archive_file_name = export.export_html(user) + except: + flash(gettext("Error when exporting articles."), 'danger') + return redirect(redirect_url()) + response = make_response(archive_file) + response.headers['Content-Type'] = 'application/x-compressed' + response.headers['Content-Disposition'] = 'attachment; filename='+archive_file_name + elif request.args.get('format') == "JSON": + # Export to JSON + try: + json_result = export.export_json(user) + except: + flash(gettext("Error when exporting articles."), 'danger') + return redirect(redirect_url()) + response = make_response(json_result) + response.mimetype = 'application/json' + response.headers["Content-Disposition"] = 'attachment; filename=account.json' + else: + flash(gettext('Export format not supported.'), 'warning') + return redirect(redirect_url()) + return response + +@app.route('/export_opml', methods=['GET']) +@login_required +def export_opml(): + """ + Export all feeds to OPML. + """ + user = User.query.filter(User.id == g.user.id).first() + response = make_response(render_template('opml.xml', user=user, now=datetime.datetime.now())) + response.headers['Content-Type'] = 'application/xml' + response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml' + return response + +@app.route('/search', methods=['GET']) +@login_required +def search(): + """ + Search articles corresponding to the query. + """ + if conf.ON_HEROKU: + flash(gettext("Full text search is not yet implemented for Heroku."), "warning") + return redirect(url_for('home')) + user = User.query.filter(User.id == g.user.id).first() + + search_result, result = [], [] + nb_articles = 0 + + query = request.args.get('query', None) + if query is not None: + try: + search_result, nb_articles = fastsearch.search(user.id, query) + except Exception as e: + flash(gettext('An error occured') + ' (%s).' % e, 'danger') + light_feed = namedtuple('Feed', ['id', 'title', 'articles'], verbose=False, rename=False) + for feed_id in search_result: + for feed in user.feeds: + if feed.id == feed_id: + articles = [] + for article_id in search_result[feed_id]: + current_article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() + articles.append(current_article) + articles = sorted(articles, key=lambda t: t.date, reverse=True) + result.append(light_feed(feed.id, feed.title, articles)) + break + return render_template('search.html', feeds=result, nb_articles=nb_articles, query=query) + +@app.route('/management', methods=['GET', 'POST']) +@login_required +def management(): + """ + Display the management page. + """ + if request.method == 'POST': + if None != request.files.get('opmlfile', None): + # Import an OPML file + data = request.files.get('opmlfile', None) + if not allowed_file(data.filename): + flash(gettext('File not allowed.'), 'danger') + else: + try: + nb = utils.import_opml(g.user.email, data.read()) + utils.fetch(g.user.email, None) + flash(str(nb) + ' ' + gettext('feeds imported.'), "success") + flash(gettext("Downloading articles..."), 'info') + except: + flash(gettext("Impossible to import the new feeds."), "danger") + elif None != request.files.get('jsonfile', None): + # Import an account + data = request.files.get('jsonfile', None) + if not allowed_file(data.filename): + flash(gettext('File not allowed.'), 'danger') + else: + try: + nb = utils.import_json(g.user.email, data.read()) + flash(gettext('Account imported.'), "success") + except: + flash(gettext("Impossible to import the account."), "danger") + else: + flash(gettext('File not allowed.'), 'danger') + + form = AddFeedForm() + nb_feeds = len(g.user.feeds.all()) + articles = Article.query.filter(Article.user_id == g.user.id) + nb_articles = articles.count() + nb_unread_articles = articles.filter(Article.readed == False).count() + return render_template('management.html', user=g.user, form=form, + nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, + not_on_heroku = not conf.ON_HEROKU) + +@app.route('/history', methods=['GET']) +@login_required +def history(): + user = User.query.filter(User.id == g.user.id).first() + return render_template('history.html') + +@app.route('/bookmarklet', methods=['GET']) +@app.route('/create_feed', methods=['GET', 'POST']) +@app.route('/edit_feed/', methods=['GET', 'POST']) +@login_required +@feed_access_required +def edit_feed(feed_id=None): + """ + Add or edit a feed. + """ + feed = Feed.query.filter(Feed.id == feed_id).first() + form = AddFeedForm() + + if request.method == 'POST': + if form.validate() == False: + return render_template('edit_feed.html', form=form) + if feed_id is not None: + # Edit an existing feed + form.populate_obj(feed) + db.session.commit() + flash(gettext('Feed successfully updated.'), 'success') + return redirect('/edit_feed/' + str(feed_id)) + else: + # Create a new feed + existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data] + if len(existing_feed) == 0: + new_feed = Feed(title=form.title.data, description="", link=form.link.data, \ + site_link=form.site_link.data, email_notification=form.email_notification.data, \ + enabled=form.enabled.data) + g.user.feeds.append(new_feed) + #user.feeds = sorted(user.feeds, key=lambda t: t.title.lower()) + db.session.commit() + flash(gettext('Feed successfully created.'), 'success') + + utils.fetch(g.user.id, Feed.query.filter(Feed.link == form.link.data).first().id) + flash(gettext("Downloading articles for the new feed..."), 'info') + + return redirect('/edit_feed/' + str(new_feed.id)) + else: + flash(gettext('Feed already in the database.'), 'warning') + return redirect('/edit_feed/' + str(existing_feed[0].id)) + + if request.method == 'GET': + if feed_id is not None: + form = AddFeedForm(obj=feed) + return render_template('edit_feed.html', action=gettext("Edit the feed"), form=form, feed=feed, \ + not_on_heroku = not conf.ON_HEROKU) + + # Enable the user to add a feed with a bookmarklet + if None is not request.args.get('url', None): + existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)] + if len(existing_feed) == 0: + g.user.feeds.append(Feed(link=request.args.get('url', None))) + db.session.commit() + return jsonify({"message":"ok"}) + return jsonify({"message":"Feed already in the database."}) + + # Return an empty form in order to create a new feed + return render_template('edit_feed.html', action=gettext("Add a feed"), form=form, \ + not_on_heroku = not conf.ON_HEROKU) + +@app.route('/delete_feed/', methods=['GET']) +@login_required +@feed_access_required +def delete_feed(feed_id=None): + """ + Delete a feed with all associated articles. + """ + feed = Feed.query.filter(Feed.id == feed_id).first() + db.session.delete(feed) + db.session.commit() + flash(gettext('Feed') + ' ' + feed.title + ' ' + gettext('successfully deleted.'), 'success') + return redirect(redirect_url()) + +@app.route('/profile', methods=['GET', 'POST']) +@login_required +def profile(): + """ + Edit the profile of the currently logged user. + """ + user = User.query.filter(User.email == g.user.email).first() + form = ProfileForm() + + if request.method == 'POST': + if form.validate(): + form.populate_obj(user) + if form.password.data != "": + user.set_password(form.password.data) + db.session.commit() + flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') + return redirect(url_for('profile')) + else: + return render_template('profile.html', form=form) + + if request.method == 'GET': + form = ProfileForm(obj=user) + return render_template('profile.html', user=user, form=form) + +@app.route('/delete_account', methods=['GET']) +@login_required +def delete_account(): + """ + Delete the account of the user (with all its data). + """ + user = User.query.filter(User.email == g.user.email).first() + if user is not None: + db.session.delete(user) + db.session.commit() + flash(gettext('Your account has been deleted.'), 'success') + else: + flash(gettext('This user does not exist.'), 'danger') + return redirect(url_for('login')) + +@app.route('/expire_articles', methods=['GET']) +@login_required +def expire_articles(): + """ + Delete articles older than the given number of weeks. + """ + current_time = datetime.datetime.utcnow() + weeks_ago = current_time - datetime.timedelta(weeks=int(request.args.get('weeks', 10))) + articles_to_delete = Article.query.filter(User.email == g.user.email, or_(Article.date < weeks_ago, Article.retrieved_date < weeks_ago)) + for article in articles_to_delete: + db.session.delete(article) + flash(gettext('Articles deleted.'), 'info') + db.session.commit() + return redirect(redirect_url()) + +@app.route('/confirm_account/', methods=['GET']) +def confirm_account(activation_key=None): + """ + Confirm the account of a user. + """ + if activation_key != "": + user = User.query.filter(User.activation_key == activation_key).first() + if user is not None: + user.activation_key = "" + db.session.commit() + flash(gettext('Your account has been confirmed.'), 'success') + else: + flash(gettext('Impossible to confirm this account.'), 'danger') + return redirect(url_for('login')) + +@app.route('/recover', methods=['GET', 'POST']) +def recover(): + """ + Enables the user to recover its account when he has forgotten + its password. + """ + import string + import random + form = RecoverPasswordForm() + + if request.method == 'POST': + if form.validate(): + user = User.query.filter(User.email == form.email.data).first() + characters = string.ascii_letters + string.digits + password = "".join(random.choice(characters) for x in range(random.randint(8, 16))) + user.set_password(password) + db.session.commit() + + # Send the confirmation email + try: + notifications.new_password_notification(user, password) + flash(gettext('New password sent to your address.'), 'success') + except Exception as e: + flash(gettext('Problem while sending your new password.') + ': ' + str(e), 'danger') + + return redirect(url_for('login')) + return render_template('recover.html', form=form) + + if request.method == 'GET': + return render_template('recover.html', form=form) + +# +# Views dedicated to administration tasks. +# +@app.route('/admin/dashboard', methods=['GET', 'POST']) +@login_required +@admin_permission.require(http_exception=403) +def dashboard(): + """ + Adminstrator's dashboard. + """ + form = InformationMessageForm() + + if request.method == 'POST': + if form.validate(): + try: + notifications.information_message(form.subject.data, form.message.data) + except Exception as e: + flash(gettext('Problem while sending email') + ': ' + str(e), 'danger') + + users = User.query.all() + return render_template('admin/dashboard.html', users=users, current_user=g.user, form=form) + +@app.route('/admin/create_user', methods=['GET', 'POST']) +@app.route('/admin/edit_user/', methods=['GET', 'POST']) +@login_required +@admin_permission.require(http_exception=403) +def create_user(user_id=None): + """ + Create or edit a user. + """ + form = ProfileForm() + + if request.method == 'POST': + if form.validate(): + role_user = Role.query.filter(Role.name == "user").first() + if user_id is not None: + # Edit a user + user = User.query.filter(User.id == user_id).first() + form.populate_obj(user) + if form.password.data != "": + user.set_password(form.password.data) + db.session.commit() + flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') + else: + # Create a new user + user = User(nickname=form.nickname.data, + email=form.email.data, + pwdhash=generate_password_hash(form.password.data)) + user.roles.extend([role_user]) + user.activation_key = "" + db.session.add(user) + db.session.commit() + flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully created.'), 'success') + return redirect("/admin/edit_user/"+str(user.id)) + else: + return redirect(url_for('create_user')) + + if request.method == 'GET': + if user_id is not None: + user = User.query.filter(User.id == user_id).first() + form = ProfileForm(obj=user) + message = gettext('Edit the user') + ' ' + user.nickname + '' + else: + form = ProfileForm() + message = gettext('Add a new user') + return render_template('/admin/create_user.html', form=form, message=message) + +@app.route('/admin/user/', methods=['GET']) +@login_required +@admin_permission.require(http_exception=403) +def user(user_id=None): + """ + See information about a user (stations, etc.). + """ + user = User.query.filter(User.id == user_id).first() + if user is not None: + return render_template('/admin/user.html', user=user) + else: + flash(gettext('This user does not exist.'), 'danger') + return redirect(redirect_url()) + +@app.route('/admin/delete_user/', methods=['GET']) +@login_required +@admin_permission.require(http_exception=403) +def delete_user(user_id=None): + """ + Delete a user (with all its data). + """ + user = User.query.filter(User.id == user_id).first() + if user is not None: + db.session.delete(user) + db.session.commit() + flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully deleted.'), 'success') + else: + flash(gettext('This user does not exist.'), 'danger') + return redirect(redirect_url()) + +@app.route('/admin/enable_user/', methods=['GET']) +@app.route('/admin/disable_user/', methods=['GET']) +@login_required +@admin_permission.require() +def disable_user(user_id=None): + """ + Enable or disable the account of a user. + """ + user = User.query.filter(User.id == user_id).first() + if user is not None: + if user.activation_key != "": + + # Send the confirmation email + try: + notifications.new_account_activation(user) + user.activation_key = "" + flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully activated.'), 'success') + except Exception as e: + flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') + + else: + import random, hashlib + user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86] + flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success') + db.session.commit() + else: + flash(gettext('This user does not exist.'), 'danger') + return redirect(redirect_url()) -- cgit From cc2829c95c391043653a3bd543176f3c92f49d72 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Thu, 15 Jan 2015 17:42:54 +0100 Subject: splitting and refactoring the Restful api part --- pyaggr3g470r/views/api.py | 356 ------------------------------------- pyaggr3g470r/views/api/__init__.py | 31 ++++ pyaggr3g470r/views/api/article.py | 125 +++++++++++++ pyaggr3g470r/views/api/common.py | 79 ++++++++ pyaggr3g470r/views/api/feed.py | 109 ++++++++++++ 5 files changed, 344 insertions(+), 356 deletions(-) delete mode 100644 pyaggr3g470r/views/api.py create mode 100644 pyaggr3g470r/views/api/__init__.py create mode 100644 pyaggr3g470r/views/api/article.py create mode 100644 pyaggr3g470r/views/api/common.py create mode 100644 pyaggr3g470r/views/api/feed.py diff --git a/pyaggr3g470r/views/api.py b/pyaggr3g470r/views/api.py deleted file mode 100644 index c220d0bc..00000000 --- a/pyaggr3g470r/views/api.py +++ /dev/null @@ -1,356 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2014/06/18 $" -__revision__ = "$Date: 2014/07/05 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "AGPLv3" - -import re -import dateutil.parser -from functools import wraps -from flask import g, Response, request, session, jsonify -from flask.ext.restful import Resource, reqparse - -import conf -if not conf.ON_HEROKU: - import pyaggr3g470r.search as fastsearch -from pyaggr3g470r import api, db -from pyaggr3g470r.models import User, Article, Feed - - -def authenticate(func): - """ - Decorator for the authentication to the web services. - """ - @wraps(func) - def wrapper(*args, **kwargs): - if not getattr(func, 'authenticated', True): - return func(*args, **kwargs) - - # authentication based on the session (already logged on the site) - if 'email' in session or g.user.is_authenticated(): - return func(*args, **kwargs) - - # authentication via HTTP only - auth = request.authorization - try: - email = auth.username - user = User.query.filter(User.email == email).first() - if user and user.check_password(auth.password) and user.activation_key == "": - g.user = user - return func(*args, **kwargs) - except AttributeError: - pass - - return Response('', 401, - {'WWWAuthenticate':'Basic realm="Login Required"'}) - return wrapper - - -def to_response(func): - def wrapper(*args, **kwargs): - res = func(*args, **kwargs) - if type(res) is tuple and len(res) == 2 and type(res[1]) is int: - response = jsonify(**res[0]) - response.status_code = res[1] - if isinstance(res, Response): - return res - else: - response = jsonify(**res) - return response - return wrapper - - -class ArticleListAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate, to_response] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('content', type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('date', type=str, location='json') - self.reqparse.add_argument('feed_id', type=int, location='json') - super(ArticleListAPI, self).__init__() - - def get(self): - """ - Returns a list of articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) - - return {'result': [article.dump() for article in articles]} - - def post(self): - """ - POST method - Create a new article. - """ - args = self.reqparse.parse_args() - article_dict = {} - for k, v in args.iteritems(): - if v != None: - article_dict[k] = v - else: - return {"message": "Missing argument: %s." % (k,)}, 400 - article_date = None - try: - article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) - except: - try: # trying to clean date field from letters - article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) - except: - return jsonify({"message": "Bad format for the date."}), 400 - article = Article(link=article_dict["link"], title=article_dict["title"], - content=article_dict["content"], readed=False, like=False, - date=article_date, user_id=g.user.id, - feed_id=article_dict["feed_id"]) - feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() - feed.articles.append(article) - try: - db.session.commit() - return {"message": "ok"}, 201 - except: - return {"message": "Impossible to create the article."}, 500 - - -class ArticleAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate, to_response] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('like', type=bool, location='json') - self.reqparse.add_argument('readed', type=bool, location= 'json') - super(ArticleAPI, self).__init__() - - def get_article_or_raise(self, article_id=None): - if article_id is None: - raise Exception({'message': 'Bad id'}, 400) - article = Article.query.filter(Article.id == article_id).first() - if article.source.subscriber.id != g.user.id: - return {'message': "Bad user for article."}, 403 - if article is None: - return {'message': 'Article not found'}, 404 - return article - - def get(self, id=None): - "Returns an article." - try: - article = self.get_article_or_raise(id) - except Exception, error: - return error.args - if not article.readed: - article.readed = True - db.session.commit() - return {'result': [article.dump()]} - - def put(self, id): - """ Update an article. It is only possible to update the status - ('like' and 'readed') of an article.""" - args = self.reqparse.parse_args() - try: - article = self.get_article_or_raise(id) - except Exception, error: - return error.args - if None is not args.get('like', None): - article.like = args['like'] - if None is not args.get('readed', None): - article.readed = args['readed'] - db.session.commit() - - try: - fastsearch.delete_article(g.user.id, article.feed_id, article.id) - except: - pass - return {"message": "ok"} - - def delete(self, id): - """ - Delete an article. - """ - try: - article = self.get_article_or_raise(id) - except Exception, error: - return error.args - db.session.delete(article) - db.session.commit() - return {"message": "ok"}, 204 - - -class FeedListAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate, to_response] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', - type=unicode, default="", location='json') - self.reqparse.add_argument('description', - type=unicode, default="", location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', - type=unicode, default="", location='json') - self.reqparse.add_argument('email_notification', - type=bool, default=False, location='json') - self.reqparse.add_argument('enabled', - type=bool, default=True, location='json') - super(FeedListAPI, self).__init__() - - def get(self): - """ - Returns a list of feeds. - """ - return {'result': [{"id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "email_notification": feed.email_notification, - "enabled": feed.enabled, - "created_date": feed.created_date, - } for feed in g.user.feeds]} - - def post(self): - """ - POST method - Create a new feed. - """ - args = self.reqparse.parse_args() - feed_dict = {} - for k, v in args.iteritems(): - if v != None: - feed_dict[k] = v - else: - return {'message': 'missing argument: %s' % (k,)}, 400 - new_feed = Feed(title=feed_dict["title"], - description=feed_dict["description"], - link=feed_dict["link"], - site_link=feed_dict["site_link"], - email_notification=feed_dict["email_notification"], - enabled=feed_dict["enabled"]) - g.user.feeds.append(new_feed) - try: - db.session.commit() - return {"message": "ok"} - except: - return {'message': 'Impossible to create the feed.'}, 500 - - -class FeedAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate, to_response] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('description', - type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', type=unicode, location='json') - self.reqparse.add_argument('email_notification', - type=bool, location='json') - self.reqparse.add_argument('enabled', type=bool ,location='json') - super(FeedAPI, self).__init__() - - def get_feed_or_raise(self, feed_id=None): - if feed_id is None: - raise Exception({'message': 'Bad id'}, 400) - feed = Article.query.filter(Article.id == feed_id).first() - if feed.source.subscriber.id != g.user.id: - return {'message': "Bad user for article."}, 403 - if feed is None: - return {'message': 'Article not found'}, 404 - return feed - - def get(self, id=None): - "Returns a feed" - try: - feed = self.get_feed_or_raise(id) - except Exception, error: - return error.args - return {'result': [feed.dump()]} - - def put(self, id): - "Update a feed" - args = self.reqparse.parse_args() - try: - feed = self.get_feed_or_raise(id) - except Exception, error: - return error.args - if 'title' in args: - feed.title = args['title'] - if 'description' in args: - feed.description = args['description'] - if 'link' in args: - feed.link = args['link'] - if 'site_link' in args: - feed.site_link = args['site_link'] - if 'email_notification' in args: - feed.email_notification = args['email_notification'] - if 'enabled' in args: - feed.enabled = args['enabled'] - db.session.commit() - return {"message": "ok"} - - def delete(self, id): - """ - Delete a feed. - """ - try: - feed = self.get_feed_or_raise(id) - except Exception, error: - return error.args - db.session.delete(feed) - db.session.commit() - return {"message": "ok"}, 204 - - -api.add_resource(ArticleListAPI, '/api/v1.0/articles', - endpoint='articles.json') -api.add_resource(ArticleAPI, '/api/v1.0/articles/', - endpoint='article.json') -api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') -api.add_resource(FeedAPI, '/api/v1.0/feeds/', endpoint = 'feed.json') diff --git a/pyaggr3g470r/views/api/__init__.py b/pyaggr3g470r/views/api/__init__.py new file mode 100644 index 00000000..e11cdd95 --- /dev/null +++ b/pyaggr3g470r/views/api/__init__.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.2 $" +__date__ = "$Date: 2014/06/18 $" +__revision__ = "$Date: 2014/07/05 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "AGPLv3" + +from pyaggr3g470r.views.api import article, feed + +__all__ = ['article', 'feed'] diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py new file mode 100644 index 00000000..c509b0a8 --- /dev/null +++ b/pyaggr3g470r/views/api/article.py @@ -0,0 +1,125 @@ +import re +import dateutil.parser + +import conf +if not conf.ON_HEROKU: + import pyaggr3g470r.search as fastsearch + +from flask import request, g +from flask.ext.restful import Resource, reqparse + +from pyaggr3g470r import api, db +from pyaggr3g470r.models import Article, Feed +from pyaggr3g470r.views.api.common import authenticate, to_response, \ + PyAggResource + + +class ArticleListAPI(Resource): + """ + Defines a RESTful API for Article elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', type=unicode, location='json') + self.reqparse.add_argument('content', type=unicode, location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('date', type=str, location='json') + self.reqparse.add_argument('feed_id', type=int, location='json') + super(ArticleListAPI, self).__init__() + + def get(self): + """ + Returns a list of articles. + """ + feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} + articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), + Article.user_id == g.user.id) + filter_ = request.args.get('filter_', 'unread') + feed_id = int(request.args.get('feed', 0)) + limit = request.args.get('limit', 1000) + if filter_ != 'all': + articles = articles.filter(Article.readed == (filter_ == 'read')) + if feed_id: + articles = articles.filter(Article.feed_id == feed_id) + + articles = articles.order_by(Article.date.desc()) + if limit != 'all': + limit = int(limit) + articles = articles.limit(limit) + + return {'result': [article.dump() for article in articles]} + + def post(self): + """ + POST method - Create a new article. + """ + args = self.reqparse.parse_args() + article_dict = {} + for k, v in args.iteritems(): + if v != None: + article_dict[k] = v + else: + return {"message": "Missing argument: %s." % (k,)}, 400 + article_date = None + try: + article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) + except: + try: # trying to clean date field from letters + article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) + except: + return {"message": "Bad format for the date."}, 400 + article = Article(link=article_dict["link"], title=article_dict["title"], + content=article_dict["content"], readed=False, like=False, + date=article_date, user_id=g.user.id, + feed_id=article_dict["feed_id"]) + feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() + feed.articles.append(article) + try: + db.session.commit() + return {"message": "ok"}, 201 + except: + return {"message": "Impossible to create the article."}, 500 + + +class ArticleAPI(PyAggResource): + "Defines a RESTful API for Article elements." + method_decorators = [authenticate, to_response] + db_cls = Article + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('like', type=bool, location='json') + self.reqparse.add_argument('readed', type=bool, location= 'json') + super(ArticleAPI, self).__init__() + + def get(self, id): + article = self._get_or_raise(id) + if not article.readed: + article.readed = True + db.session.commit() + return {'result': [article.dump()]} + + def put(self, id): + """ Update an article. It is only possible to update the status + ('like' and 'readed') of an article.""" + args = self.reqparse.parse_args() + article = self._get_or_raise(id) + if 'like' in args: + article.like = args['like'] + if 'readed' in args: + article.readed = args['readed'] + db.session.commit() + + try: + fastsearch.delete_article(g.user.id, article.feed_id, article.id) + except: + pass + return {"message": "ok"} + + +api.add_resource(ArticleListAPI, '/api/v1.0/articles', + endpoint='articles.json') +api.add_resource(ArticleAPI, '/api/v1.0/articles/', + endpoint='article.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py new file mode 100644 index 00000000..872c4ae1 --- /dev/null +++ b/pyaggr3g470r/views/api/common.py @@ -0,0 +1,79 @@ +from functools import wraps +from flask import request, g, session, Response, jsonify +from flask.ext.restful import Resource + +from pyaggr3g470r import db +from pyaggr3g470r.models import User + + +class HttpError(Exception): + pass + + +def authenticate(func): + """ + Decorator for the authentication to the web services. + """ + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + # authentication based on the session (already logged on the site) + if 'email' in session or g.user.is_authenticated(): + return func(*args, **kwargs) + + # authentication via HTTP only + auth = request.authorization + try: + email = auth.username + user = User.query.filter(User.email == email).first() + if user and user.check_password(auth.password) and user.activation_key == "": + g.user = user + return func(*args, **kwargs) + except AttributeError: + pass + + return Response('', 401, + {'WWWAuthenticate':'Basic realm="Login Required"'}) + return wrapper + + +def to_response(func): + def wrapper(*args, **kwargs): + try: + res = func(*args, **kwargs) + except HttpError, error: + return Response(*error.args) + if isinstance(res, tuple): + response = jsonify(**res[0]) + if len(res) > 1: + response.status_code = res[1] + return response + return res + return wrapper + + +class PyAggResource(Resource): + db_cls = None + + def _get_or_raise(self, obj_id=None): + if obj_id is None: + raise HttpError({'message': 'No id given'}, 400) + obj = self.db_cls.query.filter(self.db_cls.id == obj_id).first() + if obj is None: + raise HttpError({'message': 'Article not found'}, 404) + if obj.user_id != g.user.id: + raise HttpError({'message': "Unauthorized for %s." + % self.db_cls.__class__.__name__}, 403) + return obj + + def get(self, id=None): + return {'result': [self._get_or_raise(id).dump()]} + + def delete(self, id): + """Delete a feed.""" + feed = self._get_or_raise(id) + db.session.delete(feed) + db.session.commit() + return {"message": "ok"}, 204 diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py new file mode 100644 index 00000000..e5937128 --- /dev/null +++ b/pyaggr3g470r/views/api/feed.py @@ -0,0 +1,109 @@ +from flask import g +from flask.ext.restful import Resource, reqparse + +from pyaggr3g470r import api, db +from pyaggr3g470r.models import Feed + +from pyaggr3g470r.views.api.common import authenticate, to_response, \ + PyAggResource + + +class FeedListAPI(Resource): + """ + Defines a RESTful API for Feed elements. + """ + method_decorators = [authenticate, to_response] + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', + type=unicode, default="", location='json') + self.reqparse.add_argument('description', + type=unicode, default="", location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('site_link', + type=unicode, default="", location='json') + self.reqparse.add_argument('email_notification', + type=bool, default=False, location='json') + self.reqparse.add_argument('enabled', + type=bool, default=True, location='json') + super(FeedListAPI, self).__init__() + + def get(self): + """ + Returns a list of feeds. + """ + return {'result': [{"id": feed.id, + "title": feed.title, + "description": feed.description, + "link": feed.link, + "site_link": feed.site_link, + "email_notification": feed.email_notification, + "enabled": feed.enabled, + "created_date": feed.created_date, + } for feed in g.user.feeds]} + + def post(self): + """ + POST method - Create a new feed. + """ + args = self.reqparse.parse_args() + feed_dict = {} + for k, v in args.iteritems(): + if v != None: + feed_dict[k] = v + else: + return {'message': 'missing argument: %s' % (k,)}, 400 + new_feed = Feed(title=feed_dict["title"], + description=feed_dict["description"], + link=feed_dict["link"], + site_link=feed_dict["site_link"], + email_notification=feed_dict["email_notification"], + enabled=feed_dict["enabled"]) + g.user.feeds.append(new_feed) + try: + db.session.commit() + return {"message": "ok"} + except: + return {'message': 'Impossible to create the feed.'}, 500 + + +class FeedAPI(PyAggResource): + "Defines a RESTful API for Feed elements." + method_decorators = [authenticate, to_response] + db_cls = Feed + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('title', type=unicode, location='json') + self.reqparse.add_argument('description', + type=unicode, location='json') + self.reqparse.add_argument('link', type=unicode, location='json') + self.reqparse.add_argument('site_link', type=unicode, location='json') + self.reqparse.add_argument('email_notification', + type=bool, location='json') + self.reqparse.add_argument('enabled', type=bool ,location='json') + super(FeedAPI, self).__init__() + + def put(self, id): + "Update a feed" + args = self.reqparse.parse_args() + feed = self.get_feed_or_raise(id) + if 'title' in args: + feed.title = args['title'] + if 'description' in args: + feed.description = args['description'] + if 'link' in args: + feed.link = args['link'] + if 'site_link' in args: + feed.site_link = args['site_link'] + if 'email_notification' in args: + feed.email_notification = args['email_notification'] + if 'enabled' in args: + feed.enabled = args['enabled'] + db.session.commit() + return {"message": "ok"} + + +api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') +api.add_resource(FeedAPI, '/api/v1.0/feeds/', endpoint = 'feed.json') -- cgit From f5c46617e377899ff61a38c80150ab53d2b33edc Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Thu, 15 Jan 2015 18:50:05 +0100 Subject: adding news fields and migrations scripts --- .gitignore | 1 + manager.py | 16 +++++++++++++++ migrations/versions/4b5c161e1ced_.py | 32 ++++++++++++++++++++++++++++++ pyaggr3g470r/__init__.py | 1 + pyaggr3g470r/models/__init__.py | 38 +++++++++++++++++++++++------------- requirements.txt | 1 + 6 files changed, 75 insertions(+), 14 deletions(-) create mode 100755 manager.py create mode 100644 migrations/versions/4b5c161e1ced_.py diff --git a/.gitignore b/.gitignore index ea272450..e0aaa490 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ pyaggr3g470r/var/indexdir/ # Virtualenv venv +build diff --git a/manager.py b/manager.py new file mode 100755 index 00000000..ca772b60 --- /dev/null +++ b/manager.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +from flask.ext.script import Manager +from flask.ext.migrate import Migrate, MigrateCommand + +from pyaggr3g470r import app, db + + +Migrate(app, db) + +manager = Manager(app) +manager.add_command('db', MigrateCommand) + + +if __name__ == '__main__': + manager.run() diff --git a/migrations/versions/4b5c161e1ced_.py b/migrations/versions/4b5c161e1ced_.py new file mode 100644 index 00000000..e740e827 --- /dev/null +++ b/migrations/versions/4b5c161e1ced_.py @@ -0,0 +1,32 @@ +"""empty message + +Revision ID: 4b5c161e1ced +Revises: None +Create Date: 2015-01-17 01:04:10.187285 + +""" + +# revision identifiers, used by Alembic. +revision = '4b5c161e1ced' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('feed', sa.Column('error_count', sa.Integer(), nullable=True)) + op.add_column('feed', sa.Column('last_error', sa.String(), nullable=True)) + op.add_column('feed', sa.Column('last_refreshed', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('refresh_rate', sa.Integer(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'refresh_rate') + op.drop_column('feed', 'last_refreshed') + op.drop_column('feed', 'last_error') + op.drop_column('feed', 'error_count') + ### end Alembic commands ### diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index 4ba54095..0138b4d3 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -24,6 +24,7 @@ app.config['RECAPTCHA_USE_SSL'] = True app.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY app.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY + if conf.ON_HEROKU: from flask_sslify import SSLify sslify = SSLify(app) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py index 2618ed73..906d1f8c 100644 --- a/pyaggr3g470r/models/__init__.py +++ b/pyaggr3g470r/models/__init__.py @@ -36,22 +36,25 @@ from flask.ext.login import UserMixin from pyaggr3g470r import db + class User(db.Model, UserMixin): """ Represent a user. """ - id = db.Column(db.Integer, primary_key = True) - nickname = db.Column(db.String(), unique = True) - email = db.Column(db.String(254), index = True, unique = True) + id = db.Column(db.Integer, primary_key=True) + nickname = db.Column(db.String(), unique=True) + email = db.Column(db.String(254), index=True, unique=True) pwdhash = db.Column(db.String()) - roles = db.relationship('Role', backref = 'user', lazy = 'dynamic') + roles = db.relationship('Role', backref='user', lazy='dynamic') activation_key = db.Column(db.String(128), default = hashlib.sha512( str(random.getrandbits(256)).encode("utf-8") ).hexdigest()[:86]) date_created = db.Column(db.DateTime(), default=datetime.now) last_seen = db.Column(db.DateTime(), default=datetime.now) - feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan') + feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic', + cascade='all,delete-orphan') + refresh_rate = db.Column(db.Integer, default=60) # in minutes @staticmethod def make_valid_nickname(nickname): @@ -92,8 +95,8 @@ class Role(db.Model): """ Represent a role. """ - id = db.Column(db.Integer, primary_key = True) - name = db.Column(db.String(), unique = True) + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(), unique=True) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) @@ -102,7 +105,7 @@ class Feed(db.Model): """ Represent a station. """ - id = db.Column(db.Integer, primary_key = True) + id = db.Column(db.Integer, primary_key=True) title = db.Column(db.String(), default="No title") description = db.Column(db.String(), default="FR") link = db.Column(db.String()) @@ -110,8 +113,12 @@ class Feed(db.Model): email_notification = db.Column(db.Boolean(), default=False) enabled = db.Column(db.Boolean(), default=True) created_date = db.Column(db.DateTime(), default=datetime.now) - articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan', - order_by=desc("Article.date")) + last_refreshed = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + last_error = db.Column(db.String(), default="") + error_count = db.Column(db.Integer(), default=0) + articles = db.relationship('Article', backref='source', lazy='dynamic', + cascade='all,delete-orphan', + order_by=desc("Article.date")) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) @@ -124,8 +131,7 @@ class Feed(db.Model): "description": self.description, "link": self.link, "site_link": self.site_link, - "nb_articles": self.articles.count(), - } + "nb_articles": self.articles.count()} class Article(db.Model): @@ -149,13 +155,17 @@ class Article(db.Model): """ Returns the previous article (older). """ - return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first() + return Article.query.filter(Article.date < self.date, + Article.feed_id == self.feed_id)\ + .order_by(desc("Article.date")).first() def next_article(self): """ Returns the next article (newer). """ - return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first() + return Article.query.filter(Article.date > self.date, + Article.feed_id == self.feed_id)\ + .order_by(asc("Article.date")).first() def __repr__(self): return json.dumps({ diff --git a/requirements.txt b/requirements.txt index 4f9a082f..b2f96657 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ Flask-WTF Flask-RESTful Flask-Babel Flask-SSLify +Flask-Migrate WTForms python-postmark whoosh -- cgit From 2849c82255b4b889c7342a0a8fa8a4aecfbe599d Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Sat, 17 Jan 2015 16:50:38 +0100 Subject: a first big refacto of the existing arch --- bootstrap.py | 23 ++- db_create.py | 10 +- manager.py | 8 +- pyaggr3g470r/__init__.py | 49 ------ pyaggr3g470r/controllers/__init__.py | 5 + pyaggr3g470r/controllers/abstract.py | 47 ++++++ pyaggr3g470r/controllers/article.py | 20 +++ pyaggr3g470r/controllers/feed.py | 6 + pyaggr3g470r/decorators.py | 23 +++ pyaggr3g470r/lib/exceptions.py | 13 ++ pyaggr3g470r/models/__init__.py | 3 +- pyaggr3g470r/notifications.py | 5 +- pyaggr3g470r/utils.py | 9 +- pyaggr3g470r/views/__init__.py | 3 + pyaggr3g470r/views/api/article.py | 40 +---- pyaggr3g470r/views/api/common.py | 47 +++--- pyaggr3g470r/views/api/feed.py | 32 +--- pyaggr3g470r/views/article.py | 53 +++++++ pyaggr3g470r/views/feed.py | 50 +++++++ pyaggr3g470r/views/views.py | 278 ++++++++++++----------------------- runserver.py | 40 ++++- 21 files changed, 423 insertions(+), 341 deletions(-) create mode 100644 pyaggr3g470r/controllers/__init__.py create mode 100644 pyaggr3g470r/controllers/abstract.py create mode 100644 pyaggr3g470r/controllers/article.py create mode 100644 pyaggr3g470r/controllers/feed.py create mode 100644 pyaggr3g470r/lib/exceptions.py create mode 100644 pyaggr3g470r/views/article.py create mode 100644 pyaggr3g470r/views/feed.py diff --git a/bootstrap.py b/bootstrap.py index d13d4799..bb1d1278 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,5 +1,6 @@ # required imports and code exection for basic functionning +import os import sys if 'threading' in sys.modules: raise Exception('threading module loaded before patching!') @@ -11,8 +12,6 @@ if not (conf.WEBSERVER_DEBUG or conf.ON_HEROKU): gevent.monkey.patch_thread() - - def set_logging(log_path, log_level=logging.INFO, log_format='%(asctime)s %(levelname)s %(message)s'): logger = logging.getLogger('pyaggr3g470r') @@ -23,3 +22,23 @@ def set_logging(log_path, log_level=logging.INFO, logger.setLevel(log_level) set_logging(conf.LOG_PATH) + +from flask import Flask +from flask.ext.sqlalchemy import SQLAlchemy + +# Create Flask application +application = Flask(__name__, static_folder='pyaggr3g470r/static', + template_folder='pyaggr3g470r/templates') +application.debug = conf.WEBSERVER_DEBUG + +# Create dummy secrey key so we can use sessions +application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) +if not application.config['SECRET_KEY']: + application.config['SECRET_KEY'] = os.urandom(12) +application.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI + +application.config['RECAPTCHA_USE_SSL'] = True +application.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY +application.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY + +db = SQLAlchemy(application) diff --git a/db_create.py b/db_create.py index 8e743515..c10e4d43 100755 --- a/db_create.py +++ b/db_create.py @@ -10,8 +10,8 @@ __license__ = "AGPLv3" import os import bootstrap +from flask import g -from pyaggr3g470r import db from pyaggr3g470r.models import User, Role from werkzeug import generate_password_hash @@ -64,8 +64,8 @@ def db_DropEverything(db): trans.commit() -db_DropEverything(db) -db.create_all() +db_DropEverything(g.db) +g.db.create_all() role_admin = Role(name="admin") role_user = Role(name="user") @@ -76,5 +76,5 @@ user1 = User(nickname="admin", activation_key="") user1.roles.extend([role_admin, role_user]) -db.session.add(user1) -db.session.commit() +g.db.session.add(user1) +g.db.session.commit() diff --git a/manager.py b/manager.py index ca772b60..e664469a 100755 --- a/manager.py +++ b/manager.py @@ -1,14 +1,12 @@ #!/usr/bin/env python - +from bootstrap import application, db from flask.ext.script import Manager from flask.ext.migrate import Migrate, MigrateCommand -from pyaggr3g470r import app, db - -Migrate(app, db) +Migrate(application, db) -manager = Manager(app) +manager = Manager(application) manager.add_command('db', MigrateCommand) diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index 0138b4d3..e69de29b 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -1,49 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -import os -from flask import Flask -from flask.ext.sqlalchemy import SQLAlchemy -from flask.ext.babel import Babel -from flask.ext.babel import format_datetime - -import conf - -# Create Flask application -app = Flask(__name__) -app.debug = conf.WEBSERVER_DEBUG - -# Create dummy secrey key so we can use sessions -app.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) -if not app.config['SECRET_KEY']: - app.config['SECRET_KEY'] = os.urandom(12) -app.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI -db = SQLAlchemy(app) - -app.config['RECAPTCHA_USE_SSL'] = True -app.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY -app.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY - - -if conf.ON_HEROKU: - from flask_sslify import SSLify - sslify = SSLify(app) - -ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) - -def allowed_file(filename): - """ - Check if the uploaded file is allowed. - """ - return '.' in filename and \ - filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS - -babel = Babel(app) - -app.jinja_env.filters['datetime'] = format_datetime - -# Views -from flask.ext.restful import Api -api = Api(app) - -from pyaggr3g470r import views diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py new file mode 100644 index 00000000..029cbf43 --- /dev/null +++ b/pyaggr3g470r/controllers/__init__.py @@ -0,0 +1,5 @@ +from .feed import FeedController +from .article import ArticleController + + +__all__ = ['FeedController', 'ArticleController'] diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py new file mode 100644 index 00000000..fe437b09 --- /dev/null +++ b/pyaggr3g470r/controllers/abstract.py @@ -0,0 +1,47 @@ +from flask import g +from pyaggr3g470r.lib.exceptions import Forbidden, NotFound + + +class AbstractController(object): + _db_cls = None + + def __init__(self, user_id): + self.user_id = user_id + + def _get(self, **filters): + if self.user_id: + filters['user_id'] = self.user_id + db_filters = [getattr(self._db_cls, key) == value + for key, value in filters.iteritems()] + return self._db_cls.query.filter(*db_filters).first() + + def get(self, **filters): + obj = self._get(**filters).first() + if not obj: + raise NotFound({'message': 'No %r (%r)' + % (self._db_cls.__class__.__name__, filters)}) + if obj.user_id != self.user_id: + raise Forbidden({'message': 'No authorized to access %r (%r)' + % (self._db_cls.__class__.__name__, filters)}) + return obj + + def create(self, **attrs): + obj = self._db_cls(**attrs) + g.db.session.commit() + return obj + + def read(self, **filters): + return self._get(**filters) + + def update(self, obj_id, **attrs): + obj = self.get(id=obj_id) + for key, values in attrs.iteritems(): + setattr(obj, key, values) + g.db.session.commit() + return obj + + def delete(self, obj_id): + obj = self.get(id=obj_id) + g.db.session.delete(obj) + g.db.session.commit() + return obj diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py new file mode 100644 index 00000000..cfaf386d --- /dev/null +++ b/pyaggr3g470r/controllers/article.py @@ -0,0 +1,20 @@ +import conf +from .abstract import AbstractController +from pyaggr3g470r.models import Article + + +class ArticleController(AbstractController): + _db_cls = Article + + def read(self, obj_id): + article = super(ArticleController, self).read(obj_id) + if not article.readed: + self.update(obj_id, readed=True) + return article + + def delete(self, obj_id): + obj = super(ArticleController, self).delete(obj_id) + if not conf.ON_HEROKU: + import pyaggr3g470r.search as fastsearch + fastsearch.delete_article(self.user_id, obj.feed_id, obj_id) + return obj diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py new file mode 100644 index 00000000..5ada1a48 --- /dev/null +++ b/pyaggr3g470r/controllers/feed.py @@ -0,0 +1,6 @@ +from .abstract import AbstractController +from pyaggr3g470r.models import Feed + + +class FeedController(AbstractController): + _db_cls = Feed diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index 3e808793..a13dacac 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -5,8 +5,11 @@ from threading import Thread from functools import wraps from flask import g, redirect, url_for, flash +from flask.ext.babel import gettext +from flask.ext.login import login_required from pyaggr3g470r.models import Feed +from pyaggr3g470r.lib.exceptions import PyAggError def async(f): @@ -37,3 +40,23 @@ def feed_access_required(func): else: return func(*args, **kwargs) return decorated + + +def handle_pyagg_error(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except PyAggError, error: + flash(gettext(error.default_message), 'warning') + return redirect(url_for('home')) + return wrapper + + +def pyagg_default_decorator(func): + @login_required + @handle_pyagg_error + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper diff --git a/pyaggr3g470r/lib/exceptions.py b/pyaggr3g470r/lib/exceptions.py new file mode 100644 index 00000000..30c71a5c --- /dev/null +++ b/pyaggr3g470r/lib/exceptions.py @@ -0,0 +1,13 @@ +class PyAggError(Exception): + status_code = None + default_message = '' + + +class Forbidden(PyAggError): + status_code = 403 + default_message = 'You do not have the rights to access that resource' + + +class NotFound(PyAggError): + status_code = 404 + default_message = 'Resource was not found' diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py index 906d1f8c..f01eb3a3 100644 --- a/pyaggr3g470r/models/__init__.py +++ b/pyaggr3g470r/models/__init__.py @@ -30,11 +30,12 @@ import re import json import random, hashlib from datetime import datetime +from flask import g from sqlalchemy import asc, desc from werkzeug import generate_password_hash, check_password_hash from flask.ext.login import UserMixin -from pyaggr3g470r import db +db = g.db class User(db.Model, UserMixin): diff --git a/pyaggr3g470r/notifications.py b/pyaggr3g470r/notifications.py index 8b6ff0c9..e351573c 100644 --- a/pyaggr3g470r/notifications.py +++ b/pyaggr3g470r/notifications.py @@ -19,10 +19,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import conf from pyaggr3g470r import utils -from pyaggr3g470r import conf from pyaggr3g470r import emails + def information_message(subject, plaintext): """ Send an information message to the users of the platform. @@ -68,4 +69,4 @@ def new_article_notification(user, feed, article): html = """\n\n%s\n\n\n%s\n\n""" % \ (feed.title + ": " + article.title, article.content) plaintext = utils.clear_string(html) - emails.send(to=user.email, bcc=conf.NOTIFICATION_EMAIL, subject=subject, plaintext=plaintext, html=html) \ No newline at end of file + emails.send(to=user.email, bcc=conf.NOTIFICATION_EMAIL, subject=subject, plaintext=plaintext, html=html) diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py index c6264106..01bcd36c 100755 --- a/pyaggr3g470r/utils.py +++ b/pyaggr3g470r/utils.py @@ -52,9 +52,10 @@ from collections import Counter from contextlib import contextmanager import conf -from pyaggr3g470r import db +from flask import g from pyaggr3g470r.models import User, Feed, Article + # regular expression to check URL url_finders = [ re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \ @@ -136,7 +137,7 @@ def import_opml(email, opml_content): return nb nb = read(subscriptions) - db.session.commit() + g.db.session.commit() return nb def import_json(email, json_content): @@ -159,7 +160,7 @@ def import_json(email, json_content): enabled=feed["enabled"]) user.feeds.append(new_feed) nb_feeds += 1 - db.session.commit() + g.db.session.commit() # Create articles for feed in json_account["result"]: @@ -179,7 +180,7 @@ def import_json(email, json_content): user_feed.articles.append(new_article) nb_articles += 1 - db.session.commit() + g.db.session.commit() return nb_feeds, nb_articles diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py index ad71048a..029dcb7d 100644 --- a/pyaggr3g470r/views/__init__.py +++ b/pyaggr3g470r/views/__init__.py @@ -1,2 +1,5 @@ from .views import * from .api import * + +from .article import article_bp, articles_bp +from .feed import feed_bp, feeds_bp diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index c509b0a8..3642cda9 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -1,15 +1,11 @@ import re import dateutil.parser -import conf -if not conf.ON_HEROKU: - import pyaggr3g470r.search as fastsearch - from flask import request, g from flask.ext.restful import Resource, reqparse -from pyaggr3g470r import api, db from pyaggr3g470r.models import Article, Feed +from pyaggr3g470r.controllers import ArticleController from pyaggr3g470r.views.api.common import authenticate, to_response, \ PyAggResource @@ -77,7 +73,7 @@ class ArticleListAPI(Resource): feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() feed.articles.append(article) try: - db.session.commit() + g.db.session.commit() return {"message": "ok"}, 201 except: return {"message": "Impossible to create the article."}, 500 @@ -86,7 +82,8 @@ class ArticleListAPI(Resource): class ArticleAPI(PyAggResource): "Defines a RESTful API for Article elements." method_decorators = [authenticate, to_response] - db_cls = Article + controller_cls = ArticleController + editable_attrs = ['like', 'readed'] def __init__(self): self.reqparse = reqparse.RequestParser() @@ -94,32 +91,7 @@ class ArticleAPI(PyAggResource): self.reqparse.add_argument('readed', type=bool, location= 'json') super(ArticleAPI, self).__init__() - def get(self, id): - article = self._get_or_raise(id) - if not article.readed: - article.readed = True - db.session.commit() - return {'result': [article.dump()]} - - def put(self, id): - """ Update an article. It is only possible to update the status - ('like' and 'readed') of an article.""" - args = self.reqparse.parse_args() - article = self._get_or_raise(id) - if 'like' in args: - article.like = args['like'] - if 'readed' in args: - article.readed = args['readed'] - db.session.commit() - - try: - fastsearch.delete_article(g.user.id, article.feed_id, article.id) - except: - pass - return {"message": "ok"} - -api.add_resource(ArticleListAPI, '/api/v1.0/articles', - endpoint='articles.json') -api.add_resource(ArticleAPI, '/api/v1.0/articles/', +g.api.add_resource(ArticleListAPI, '/articles', endpoint='articles.json') +g.api.add_resource(ArticleAPI, '/articles/', endpoint='article.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index 872c4ae1..a116b9c3 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -2,12 +2,8 @@ from functools import wraps from flask import request, g, session, Response, jsonify from flask.ext.restful import Resource -from pyaggr3g470r import db from pyaggr3g470r.models import User - - -class HttpError(Exception): - pass +from pyaggr3g470r.lib.exceptions import PyAggError def authenticate(func): @@ -43,8 +39,10 @@ def to_response(func): def wrapper(*args, **kwargs): try: res = func(*args, **kwargs) - except HttpError, error: - return Response(*error.args) + except PyAggError, error: + response = jsonify(**error.message) + response.status_code = error.status_code + return response if isinstance(res, tuple): response = jsonify(**res[0]) if len(res) > 1: @@ -55,25 +53,24 @@ def to_response(func): class PyAggResource(Resource): - db_cls = None + method_decorators = [authenticate, to_response] + controller_cls = None + editable_attrs = [] + + def __init__(self, *args, **kwargs): + self.controller = self.controller_cls(g.user.id) + super(PyAggResource, self).__init__(*args, **kwargs) - def _get_or_raise(self, obj_id=None): - if obj_id is None: - raise HttpError({'message': 'No id given'}, 400) - obj = self.db_cls.query.filter(self.db_cls.id == obj_id).first() - if obj is None: - raise HttpError({'message': 'Article not found'}, 404) - if obj.user_id != g.user.id: - raise HttpError({'message': "Unauthorized for %s." - % self.db_cls.__class__.__name__}, 403) - return obj + def get(self, obj_id=None): + return {'result': [self.controller.read(id=obj_id).dump()]} - def get(self, id=None): - return {'result': [self._get_or_raise(id).dump()]} + def put(self, obj_id=None): + args = self.reqparse.parse_args() + new_values = {key: args[key] for key in + set(args).intersection(self.editable_attrs)} + self.controller.update(obj_id, **new_values) + return {"message": "ok"} - def delete(self, id): - """Delete a feed.""" - feed = self._get_or_raise(id) - db.session.delete(feed) - db.session.commit() + def delete(self, obj_id=None): + self.controller.delete(obj_id) return {"message": "ok"}, 204 diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index e5937128..346898b5 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,7 +1,7 @@ from flask import g from flask.ext.restful import Resource, reqparse -from pyaggr3g470r import api, db +from pyaggr3g470r.controllers import FeedController from pyaggr3g470r.models import Feed from pyaggr3g470r.views.api.common import authenticate, to_response, \ @@ -62,7 +62,7 @@ class FeedListAPI(Resource): enabled=feed_dict["enabled"]) g.user.feeds.append(new_feed) try: - db.session.commit() + g.db.session.commit() return {"message": "ok"} except: return {'message': 'Impossible to create the feed.'}, 500 @@ -70,8 +70,9 @@ class FeedListAPI(Resource): class FeedAPI(PyAggResource): "Defines a RESTful API for Feed elements." - method_decorators = [authenticate, to_response] - db_cls = Feed + controller_cls = FeedController + editable_attrs = ['title', 'description', 'link', 'site_link', + 'email_notification', 'enabled'] def __init__(self): self.reqparse = reqparse.RequestParser() @@ -85,25 +86,6 @@ class FeedAPI(PyAggResource): self.reqparse.add_argument('enabled', type=bool ,location='json') super(FeedAPI, self).__init__() - def put(self, id): - "Update a feed" - args = self.reqparse.parse_args() - feed = self.get_feed_or_raise(id) - if 'title' in args: - feed.title = args['title'] - if 'description' in args: - feed.description = args['description'] - if 'link' in args: - feed.link = args['link'] - if 'site_link' in args: - feed.site_link = args['site_link'] - if 'email_notification' in args: - feed.email_notification = args['email_notification'] - if 'enabled' in args: - feed.enabled = args['enabled'] - db.session.commit() - return {"message": "ok"} - -api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') -api.add_resource(FeedAPI, '/api/v1.0/feeds/', endpoint = 'feed.json') +g.api.add_resource(FeedListAPI, '/feeds', endpoint='feeds.json') +g.api.add_resource(FeedAPI, '/feeds/', endpoint='feed.json') diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py new file mode 100644 index 00000000..21858a33 --- /dev/null +++ b/pyaggr3g470r/views/article.py @@ -0,0 +1,53 @@ +from flask import Blueprint, g, render_template, redirect +from sqlalchemy import desc + +from pyaggr3g470r import controllers, utils +from pyaggr3g470r.decorators import pyagg_default_decorator + +articles_bp = Blueprint('articles', __name__, url_prefix='/articles') +article_bp = Blueprint('article', __name__, url_prefix='/article') + + +@articles_bp.route('/', methods=['GET']) +@articles_bp.route('//', methods=['GET']) +@pyagg_default_decorator +def articles(feed_id=None, nb_articles=-1): + """List articles of a feed. The administrator of the platform is able to + access to this view for every users.""" + feed = controllers.FeedController(g.user.id).get(id=feed_id) + feed.articles = controllers.ArticleController(g.user.id)\ + .read(feed_id=feed.id)\ + .order_by(desc("Article.date")) + if len(feed.articles.all()) <= nb_articles: + nb_articles = -1 + if nb_articles == -1: + feed.articles = feed.article.limit(nb_articles) + return render_template('articles.html', feed=feed, nb_articles=nb_articles) + + +@article_bp.route('/redirect/', methods=['GET']) +@pyagg_default_decorator +def redirect_to_article(article_id): + article = controllers.ArticleController(g.user.id).get(id=article_id) + return redirect(article.link) + + +@article_bp.route('/', methods=['GET']) +@pyagg_default_decorator +def article(article_id=None): + """ + Presents the content of an article. + """ + article = controllers.ArticleController(g.user.id).get(id=article_id) + previous_article = article.previous_article() + if previous_article is None: + previous_article = article.source.articles[0] + next_article = article.next_article() + if next_article is None: + next_article = article.source.articles[-1] + + return render_template('article.html', + head_title=utils.clear_string(article.title), + article=article, + previous_article=previous_article, + next_article=next_article) diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py new file mode 100644 index 00000000..fa9dc23d --- /dev/null +++ b/pyaggr3g470r/views/feed.py @@ -0,0 +1,50 @@ +from datetime import datetime +from flask import Blueprint, g, render_template + +from pyaggr3g470r import controllers, utils +from pyaggr3g470r.decorators import pyagg_default_decorator, \ + feed_access_required + +feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds') +feed_bp = Blueprint('feed', __name__, url_prefix='/feed') + +@feeds_bp.route('/', methods=['GET']) +def feeds(): + "Lists the subscribed feeds in a table." + return render_template('feeds.html', + feeds=controllers.FeedController(g.user.id).read()) + + +@feed_bp.route('/', methods=['GET']) +@pyagg_default_decorator +@feed_access_required +def feed(feed_id=None): + "Presents detailed information about a feed." + feed = controllers.FeedController(g.user.id).get(id=feed_id) + word_size = 6 + articles = controllers.ArticleController(g.user.id)\ + .read(feed_id=feed_id).all() + nb_articles = controllers.ArticleController(g.user.id).read().count() + top_words = utils.top_words(articles, n=50, size=int(word_size)) + tag_cloud = utils.tag_cloud(top_words) + + today = datetime.datetime.now() + try: + last_article = articles[0].date + first_article = articles[-1].date + delta = last_article - first_article + average = round(float(len(articles)) / abs(delta.days), 2) + except: + last_article = datetime.datetime.fromtimestamp(0) + first_article = datetime.datetime.fromtimestamp(0) + delta = last_article - first_article + average = 0 + elapsed = today - last_article + + return render_template('feed.html', + head_title=utils.clear_string(feed.title), + feed=feed, tag_cloud=tag_cloud, + first_post_date=first_article, + end_post_date=last_article, + nb_articles=nb_articles, + average=average, delta=delta, elapsed=elapsed) diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 0a50d8d0..6e467fb0 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -28,9 +28,12 @@ __license__ = "AGPLv3" import os import json +import string +import random +import hashlib import datetime from collections import namedtuple -from flask import abort, render_template, request, flash, session, Response, \ +from flask import render_template, request, flash, session, Response, \ url_for, redirect, g, current_app, make_response, jsonify from flask.ext.login import LoginManager, login_user, logout_user, \ login_required, current_user, AnonymousUserMixin @@ -38,13 +41,12 @@ from flask.ext.principal import Principal, Identity, AnonymousIdentity, \ identity_changed, identity_loaded, Permission,\ RoleNeed, UserNeed from flask.ext.babel import gettext -from sqlalchemy import desc, func, or_ +from sqlalchemy import func, or_ from sqlalchemy.exc import IntegrityError from werkzeug import generate_password_hash import conf from pyaggr3g470r import utils, notifications, export, duplicate -from pyaggr3g470r import app, db, allowed_file, babel from pyaggr3g470r.models import User, Feed, Article, Role from pyaggr3g470r.decorators import feed_access_required from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \ @@ -53,17 +55,17 @@ if not conf.ON_HEROKU: import pyaggr3g470r.search as fastsearch -Principal(app) +Principal(g.app) # Create a permission with a single Need, in this case a RoleNeed. admin_permission = Permission(RoleNeed('admin')) login_manager = LoginManager() -login_manager.init_app(app) +login_manager.init_app(g.app) # # Management of the user's session. # -@identity_loaded.connect_via(app) +@identity_loaded.connect_via(g.app) def on_identity_loaded(sender, identity): # Set the identity user object identity.user = current_user @@ -78,13 +80,13 @@ def on_identity_loaded(sender, identity): for role in current_user.roles: identity.provides.add(RoleNeed(role.name)) -@app.before_request +@g.app.before_request def before_request(): g.user = current_user if g.user.is_authenticated(): g.user.last_seen = datetime.datetime.utcnow() - db.session.add(g.user) - db.session.commit() + g.db.session.add(g.user) + g.db.session.commit() @login_manager.user_loader def load_user(email): @@ -95,21 +97,21 @@ def load_user(email): # # Custom error pages. # -@app.errorhandler(401) +@g.app.errorhandler(401) def authentication_required(e): flash(gettext('Authentication required.'), 'info') return redirect(url_for('login')) -@app.errorhandler(403) +@g.app.errorhandler(403) def authentication_failed(e): flash(gettext('Forbidden.'), 'danger') return redirect(url_for('home')) -@app.errorhandler(404) +@g.app.errorhandler(404) def page_not_found(e): return render_template('errors/404.html'), 404 -@app.errorhandler(500) +@g.app.errorhandler(500) def internal_server_error(e): return render_template('errors/500.html'), 500 @@ -119,7 +121,7 @@ def redirect_url(default='home'): request.referrer or \ url_for(default) -@babel.localeselector +@g.babel.localeselector def get_locale(): """ Called before each request to give us a chance to choose @@ -127,7 +129,7 @@ def get_locale(): """ return request.accept_languages.best_match(conf.LANGUAGES.keys()) -@babel.timezoneselector +@g.babel.timezoneselector def get_timezone(): try: return conf.TIME_ZONE[get_locale()] @@ -137,7 +139,7 @@ def get_timezone(): # # Views. # -@app.route('/login', methods=['GET', 'POST']) +@g.app.route('/login', methods=['GET', 'POST']) def login(): """ Log in view. @@ -158,7 +160,7 @@ def login(): return redirect(url_for('home')) return render_template('login.html', form=form) -@app.route('/api/csrf', methods=['GET']) +@g.app.route('/api/csrf', methods=['GET']) def get_csrf(): try: data = json.loads(request.data) @@ -170,11 +172,11 @@ def get_csrf(): return Response(status=401) user = User.query.filter(User.email == email).first() if not user: - return Reponse(status=404) + return Response(status=404) if not user.check_password(password): - return Reponse(status=401) + return Response(status=401) if not user.activation_key == "": - return Reponse(status=403) + return Response(status=403) login_user(user) g.user = user session['email'] = email @@ -183,7 +185,7 @@ def get_csrf(): return 'ok', 200 -@app.route('/logout') +@g.app.route('/logout') @login_required def logout(): """ @@ -204,7 +206,7 @@ def logout(): flash(gettext("Logged out successfully."), 'success') return redirect(url_for('login')) -@app.route('/signup', methods=['GET', 'POST']) +@g.app.route('/signup', methods=['GET', 'POST']) def signup(): """ Signup page. @@ -223,9 +225,9 @@ def signup(): email=form.email.data, pwdhash=generate_password_hash(form.password.data)) user.roles = [role_user] - db.session.add(user) + g.db.session.add(user) try: - db.session.commit() + g.db.session.commit() except IntegrityError: flash(gettext('Email already used.'), 'warning') return render_template('signup.html', form=form) @@ -242,7 +244,7 @@ def signup(): return render_template('signup.html', form=form) -@app.route('/') +@g.app.route('/') @login_required def home(): """ @@ -263,7 +265,7 @@ def home(): if limit != 'all': limit = int(limit) articles = articles.limit(limit) - unread = db.session.query(Article.feed_id, func.count(Article.id))\ + unread = g.db.session.query(Article.feed_id, func.count(Article.id))\ .filter(Article.readed == False, Article.user_id == g.user.id)\ .group_by(Article.feed_id).all() def gen_url(filter_=filter_, limit=limit, feed=feed_id): @@ -273,20 +275,8 @@ def home(): unread=dict(unread), articles=articles.all()) -@app.route('/article/redirect/', methods=['GET']) -@login_required -def redirect_to_article(article_id): - article = Article.query.filter(Article.id == article_id, - Article.user_id == g.user.id).first() - if article is None: - abort(404) - article.readed = True - db.session.commit() - return redirect(article.link) - - -@app.route('/fetch', methods=['GET']) -@app.route('/fetch/', methods=['GET']) +@g.app.route('/fetch', methods=['GET']) +@g.app.route('/fetch/', methods=['GET']) @login_required def fetch(feed_id=None): """ @@ -297,82 +287,17 @@ def fetch(feed_id=None): flash(gettext("Downloading articles..."), 'info') return redirect(redirect_url()) -@app.route('/about', methods=['GET']) +@g.app.route('/about', methods=['GET']) def about(): """ 'About' page. """ return render_template('about.html') -@app.route('/feeds', methods=['GET']) -@login_required -def feeds(): - """ - Lists the subscribed feeds in a table. - """ - user = User.query.filter(User.email == g.user.email).first() - return render_template('feeds.html', feeds=user.feeds) - -@app.route('/feed/', methods=['GET']) -@login_required -@feed_access_required -def feed(feed_id=None): - """ - Presents detailed information about a feed. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - word_size = 6 - articles = feed.articles.all() - nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all()) - top_words = utils.top_words(articles, n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) - - today = datetime.datetime.now() - try: - last_article = articles[0].date - first_article = articles[-1].date - delta = last_article - first_article - average = round(float(len(articles)) / abs(delta.days), 2) - except: - last_article = datetime.datetime.fromtimestamp(0) - first_article = datetime.datetime.fromtimestamp(0) - delta = last_article - first_article - average = 0 - elapsed = today - last_article - - return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \ - first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \ - average=average, delta=delta, elapsed=elapsed) - -@app.route('/article/', methods=['GET']) -@login_required -def article(article_id=None): - """ - Presents the content of an article. - """ - article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() - if article is not None: - if not article.readed: - article.readed = True - db.session.commit() - - previous_article = article.previous_article() - if previous_article is None: - previous_article = article.source.articles[0] - next_article = article.next_article() - if next_article is None: - next_article = article.source.articles[-1] - - return render_template('article.html', head_title=utils.clear_string(article.title), - article=article, - previous_article=previous_article, next_article=next_article) - flash(gettext("This article do not exist."), 'warning') - return redirect(url_for('home')) - -@app.route('/mark_as/', methods=['GET']) -@app.route('/mark_as//feed/', methods=['GET']) -@app.route('/mark_as//article/', methods=['GET']) +@g.app.route('/mark_as/', methods=['GET']) +@g.app.route('/mark_as//feed/', methods=['GET']) +@g.app.route('/mark_as//article/', methods=['GET']) @login_required @feed_access_required def mark_as(new_value='read', feed_id=None, article_id=None): @@ -391,12 +316,12 @@ def mark_as(new_value='read', feed_id=None, article_id=None): message = 'All article marked as %s.' articles.filter(Article.readed == (not readed)).update({"readed": readed}) flash(gettext(message % new_value), 'info') - db.session.commit() + g.db.session.commit() if readed: return redirect(redirect_url()) return redirect(url_for('home')) -@app.route('/like/', methods=['GET']) +@g.app.route('/like/', methods=['GET']) @login_required def like(article_id=None): """ @@ -406,10 +331,10 @@ def like(article_id=None): update({ "like": not Article.query.filter(Article.id == article_id).first().like }) - db.session.commit() + g.db.session.commit() return redirect(redirect_url()) -@app.route('/delete/', methods=['GET']) +@g.app.route('/delete/', methods=['GET']) @login_required def delete(article_id=None): """ @@ -417,8 +342,8 @@ def delete(article_id=None): """ article = Article.query.filter(Article.id == article_id).first() if article is not None and article.source.subscriber.id == g.user.id: - db.session.delete(article) - db.session.commit() + g.db.session.delete(article) + g.db.session.commit() try: fastsearch.delete_article(g.user.id, article.feed_id, article.id) except: @@ -429,26 +354,8 @@ def delete(article_id=None): flash(gettext('This article do not exist.'), 'danger') return redirect(url_for('home')) -@app.route('/articles/', methods=['GET']) -@app.route('/articles//', methods=['GET']) -@login_required -@feed_access_required -def articles(feed_id=None, nb_articles=-1): - """ - List articles of a feed. - The administrator of the platform is able to access to this view for every users. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - new_feed = feed - if len(feed.articles.all()) <= nb_articles: - nb_articles = -1 - if nb_articles == -1: - nb_articles = int(1e9) - new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \ - Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles) - return render_template('articles.html', feed=new_feed, nb_articles=nb_articles) - -@app.route('/favorites', methods=['GET']) + +@g.app.route('/favorites', methods=['GET']) @login_required def favorites(): """ @@ -463,8 +370,8 @@ def favorites(): nb_favorites += len(articles) return render_template('favorites.html', feeds=result, nb_favorites=nb_favorites) -@app.route('/unread/', methods=['GET']) -@app.route('/unread', methods=['GET']) +@g.app.route('/unread/', methods=['GET']) +@g.app.route('/unread', methods=['GET']) @login_required def unread(feed_id=None): """ @@ -482,7 +389,7 @@ def unread(feed_id=None): nb_unread += len(articles) return render_template('unread.html', feeds=result, nb_unread=nb_unread) -@app.route('/inactives', methods=['GET']) +@g.app.route('/inactives', methods=['GET']) @login_required def inactives(): """ @@ -502,7 +409,7 @@ def inactives(): inactives.append((feed, elapsed)) return render_template('inactives.html', inactives=inactives, nb_days=nb_days) -@app.route('/duplicates/', methods=['GET']) +@g.app.route('/duplicates/', methods=['GET']) @login_required def duplicates(feed_id=None): """ @@ -513,7 +420,7 @@ def duplicates(feed_id=None): duplicates = duplicate.compare_documents(feed) return render_template('duplicates.html', duplicates=duplicates, feed=feed) -@app.route('/index_database', methods=['GET']) +@g.app.route('/index_database', methods=['GET']) @login_required def index_database(): """ @@ -530,7 +437,7 @@ def index_database(): flash(gettext('Option not available on Heroku.'), 'success') return redirect(url_for('home')) -@app.route('/export', methods=['GET']) +@g.app.route('/export', methods=['GET']) @login_required def export_articles(): """ @@ -562,7 +469,7 @@ def export_articles(): return redirect(redirect_url()) return response -@app.route('/export_opml', methods=['GET']) +@g.app.route('/export_opml', methods=['GET']) @login_required def export_opml(): """ @@ -574,7 +481,7 @@ def export_opml(): response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml' return response -@app.route('/search', methods=['GET']) +@g.app.route('/search', methods=['GET']) @login_required def search(): """ @@ -607,7 +514,7 @@ def search(): break return render_template('search.html', feeds=result, nb_articles=nb_articles, query=query) -@app.route('/management', methods=['GET', 'POST']) +@g.app.route('/management', methods=['GET', 'POST']) @login_required def management(): """ @@ -617,7 +524,7 @@ def management(): if None != request.files.get('opmlfile', None): # Import an OPML file data = request.files.get('opmlfile', None) - if not allowed_file(data.filename): + if not g.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: @@ -630,7 +537,7 @@ def management(): elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) - if not allowed_file(data.filename): + if not g.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: @@ -650,15 +557,15 @@ def management(): nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, not_on_heroku = not conf.ON_HEROKU) -@app.route('/history', methods=['GET']) +@g.app.route('/history', methods=['GET']) @login_required def history(): - user = User.query.filter(User.id == g.user.id).first() + #user = User.query.filter(User.id == g.user.id).first() return render_template('history.html') -@app.route('/bookmarklet', methods=['GET']) -@app.route('/create_feed', methods=['GET', 'POST']) -@app.route('/edit_feed/', methods=['GET', 'POST']) +@g.app.route('/bookmarklet', methods=['GET']) +@g.app.route('/create_feed', methods=['GET', 'POST']) +@g.app.route('/edit_feed/', methods=['GET', 'POST']) @login_required @feed_access_required def edit_feed(feed_id=None): @@ -674,19 +581,19 @@ def edit_feed(feed_id=None): if feed_id is not None: # Edit an existing feed form.populate_obj(feed) - db.session.commit() + g.db.session.commit() flash(gettext('Feed successfully updated.'), 'success') return redirect('/edit_feed/' + str(feed_id)) else: # Create a new feed - existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data] + existing_feed = [f for f in g.user.feeds if feed.link == form.link.data] if len(existing_feed) == 0: new_feed = Feed(title=form.title.data, description="", link=form.link.data, \ site_link=form.site_link.data, email_notification=form.email_notification.data, \ enabled=form.enabled.data) g.user.feeds.append(new_feed) #user.feeds = sorted(user.feeds, key=lambda t: t.title.lower()) - db.session.commit() + g.db.session.commit() flash(gettext('Feed successfully created.'), 'success') utils.fetch(g.user.id, Feed.query.filter(Feed.link == form.link.data).first().id) @@ -705,10 +612,10 @@ def edit_feed(feed_id=None): # Enable the user to add a feed with a bookmarklet if None is not request.args.get('url', None): - existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)] + existing_feed = [f for f in g.user.feeds if feed.link == request.args.get('url', None)] if len(existing_feed) == 0: g.user.feeds.append(Feed(link=request.args.get('url', None))) - db.session.commit() + g.db.session.commit() return jsonify({"message":"ok"}) return jsonify({"message":"Feed already in the database."}) @@ -716,7 +623,7 @@ def edit_feed(feed_id=None): return render_template('edit_feed.html', action=gettext("Add a feed"), form=form, \ not_on_heroku = not conf.ON_HEROKU) -@app.route('/delete_feed/', methods=['GET']) +@g.app.route('/delete_feed/', methods=['GET']) @login_required @feed_access_required def delete_feed(feed_id=None): @@ -724,12 +631,12 @@ def delete_feed(feed_id=None): Delete a feed with all associated articles. """ feed = Feed.query.filter(Feed.id == feed_id).first() - db.session.delete(feed) - db.session.commit() + g.db.session.delete(feed) + g.db.session.commit() flash(gettext('Feed') + ' ' + feed.title + ' ' + gettext('successfully deleted.'), 'success') return redirect(redirect_url()) -@app.route('/profile', methods=['GET', 'POST']) +@g.app.route('/profile', methods=['GET', 'POST']) @login_required def profile(): """ @@ -743,7 +650,7 @@ def profile(): form.populate_obj(user) if form.password.data != "": user.set_password(form.password.data) - db.session.commit() + g.db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') return redirect(url_for('profile')) else: @@ -753,7 +660,7 @@ def profile(): form = ProfileForm(obj=user) return render_template('profile.html', user=user, form=form) -@app.route('/delete_account', methods=['GET']) +@g.app.route('/delete_account', methods=['GET']) @login_required def delete_account(): """ @@ -761,14 +668,14 @@ def delete_account(): """ user = User.query.filter(User.email == g.user.email).first() if user is not None: - db.session.delete(user) - db.session.commit() + g.db.session.delete(user) + g.db.session.commit() flash(gettext('Your account has been deleted.'), 'success') else: flash(gettext('This user does not exist.'), 'danger') return redirect(url_for('login')) -@app.route('/expire_articles', methods=['GET']) +@g.app.route('/expire_articles', methods=['GET']) @login_required def expire_articles(): """ @@ -778,12 +685,12 @@ def expire_articles(): weeks_ago = current_time - datetime.timedelta(weeks=int(request.args.get('weeks', 10))) articles_to_delete = Article.query.filter(User.email == g.user.email, or_(Article.date < weeks_ago, Article.retrieved_date < weeks_ago)) for article in articles_to_delete: - db.session.delete(article) + g.db.session.delete(article) flash(gettext('Articles deleted.'), 'info') - db.session.commit() + g.db.session.commit() return redirect(redirect_url()) -@app.route('/confirm_account/', methods=['GET']) +@g.app.route('/confirm_account/', methods=['GET']) def confirm_account(activation_key=None): """ Confirm the account of a user. @@ -792,20 +699,18 @@ def confirm_account(activation_key=None): user = User.query.filter(User.activation_key == activation_key).first() if user is not None: user.activation_key = "" - db.session.commit() + g.db.session.commit() flash(gettext('Your account has been confirmed.'), 'success') else: flash(gettext('Impossible to confirm this account.'), 'danger') return redirect(url_for('login')) -@app.route('/recover', methods=['GET', 'POST']) +@g.app.route('/recover', methods=['GET', 'POST']) def recover(): """ Enables the user to recover its account when he has forgotten its password. """ - import string - import random form = RecoverPasswordForm() if request.method == 'POST': @@ -814,7 +719,7 @@ def recover(): characters = string.ascii_letters + string.digits password = "".join(random.choice(characters) for x in range(random.randint(8, 16))) user.set_password(password) - db.session.commit() + g.db.session.commit() # Send the confirmation email try: @@ -832,7 +737,7 @@ def recover(): # # Views dedicated to administration tasks. # -@app.route('/admin/dashboard', methods=['GET', 'POST']) +@g.app.route('/admin/dashboard', methods=['GET', 'POST']) @login_required @admin_permission.require(http_exception=403) def dashboard(): @@ -851,8 +756,8 @@ def dashboard(): users = User.query.all() return render_template('admin/dashboard.html', users=users, current_user=g.user, form=form) -@app.route('/admin/create_user', methods=['GET', 'POST']) -@app.route('/admin/edit_user/', methods=['GET', 'POST']) +@g.app.route('/admin/create_user', methods=['GET', 'POST']) +@g.app.route('/admin/edit_user/', methods=['GET', 'POST']) @login_required @admin_permission.require(http_exception=403) def create_user(user_id=None): @@ -870,7 +775,7 @@ def create_user(user_id=None): form.populate_obj(user) if form.password.data != "": user.set_password(form.password.data) - db.session.commit() + g.db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') else: # Create a new user @@ -879,8 +784,8 @@ def create_user(user_id=None): pwdhash=generate_password_hash(form.password.data)) user.roles.extend([role_user]) user.activation_key = "" - db.session.add(user) - db.session.commit() + g.db.session.add(user) + g.db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully created.'), 'success') return redirect("/admin/edit_user/"+str(user.id)) else: @@ -896,7 +801,7 @@ def create_user(user_id=None): message = gettext('Add a new user') return render_template('/admin/create_user.html', form=form, message=message) -@app.route('/admin/user/', methods=['GET']) +@g.app.route('/admin/user/', methods=['GET']) @login_required @admin_permission.require(http_exception=403) def user(user_id=None): @@ -910,7 +815,7 @@ def user(user_id=None): flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) -@app.route('/admin/delete_user/', methods=['GET']) +@g.app.route('/admin/delete_user/', methods=['GET']) @login_required @admin_permission.require(http_exception=403) def delete_user(user_id=None): @@ -919,15 +824,15 @@ def delete_user(user_id=None): """ user = User.query.filter(User.id == user_id).first() if user is not None: - db.session.delete(user) - db.session.commit() + g.db.session.delete(user) + g.db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully deleted.'), 'success') else: flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) -@app.route('/admin/enable_user/', methods=['GET']) -@app.route('/admin/disable_user/', methods=['GET']) +@g.app.route('/admin/enable_user/', methods=['GET']) +@g.app.route('/admin/disable_user/', methods=['GET']) @login_required @admin_permission.require() def disable_user(user_id=None): @@ -947,10 +852,9 @@ def disable_user(user_id=None): flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') else: - import random, hashlib user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86] flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success') - db.session.commit() + g.db.session.commit() else: flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) diff --git a/runserver.py b/runserver.py index 8ae7282a..8d163cd6 100755 --- a/runserver.py +++ b/runserver.py @@ -19,8 +19,44 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from bootstrap import conf -from pyaggr3g470r import app as application +from bootstrap import conf, application, db +from flask.ext.babel import Babel +from flask.ext.babel import format_datetime + +if conf.ON_HEROKU: + from flask_sslify import SSLify + SSLify(application) + +ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) + +def allowed_file(filename): + """ + Check if the uploaded file is allowed. + """ + return '.' in filename and \ + filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS + +babel = Babel(application) + +application.jinja_env.filters['datetime'] = format_datetime + +# Views +from flask.ext.restful import Api +from flask import g + +with application.app_context(): + g.api = Api(application, prefix='/api/v1.0') + g.babel = babel + g.allowed_file = allowed_file + g.db = db + g.app = application + + from pyaggr3g470r import views + application.register_blueprint(views.articles_bp) + application.register_blueprint(views.article_bp) + application.register_blueprint(views.feeds_bp) + application.register_blueprint(views.feed_bp) + if __name__ == '__main__': application.run(host=conf.WEBSERVER_HOST, -- cgit From 5ce0ce0d57c9d9976a47a120ca6235b84ade236a Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 20 Jan 2015 17:20:07 +0100 Subject: first implementation of fetchable feeds --- pyaggr3g470r/controllers/__init__.py | 3 +- pyaggr3g470r/controllers/abstract.py | 43 ++++++---- pyaggr3g470r/controllers/feed.py | 17 ++++ pyaggr3g470r/controllers/user.py | 7 ++ pyaggr3g470r/views/api/common.py | 2 +- pyaggr3g470r/views/api/feed.py | 24 +++++- pyaggr3g470r/views/views.py | 151 ++++++++++++++++++----------------- 7 files changed, 153 insertions(+), 94 deletions(-) create mode 100644 pyaggr3g470r/controllers/user.py diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py index 029cbf43..d8d1a104 100644 --- a/pyaggr3g470r/controllers/__init__.py +++ b/pyaggr3g470r/controllers/__init__.py @@ -1,5 +1,6 @@ from .feed import FeedController from .article import ArticleController +from .user import UserController -__all__ = ['FeedController', 'ArticleController'] +__all__ = ['FeedController', 'ArticleController', 'UserController'] diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py index fe437b09..8960c3be 100644 --- a/pyaggr3g470r/controllers/abstract.py +++ b/pyaggr3g470r/controllers/abstract.py @@ -1,47 +1,60 @@ -from flask import g +from bootstrap import db from pyaggr3g470r.lib.exceptions import Forbidden, NotFound class AbstractController(object): _db_cls = None + _user_id_key = 'user_id' def __init__(self, user_id): self.user_id = user_id def _get(self, **filters): if self.user_id: - filters['user_id'] = self.user_id - db_filters = [getattr(self._db_cls, key) == value - for key, value in filters.iteritems()] - return self._db_cls.query.filter(*db_filters).first() + filters[self._user_id_key] = self.user_id + db_filters = set() + for key, value in filters.iteritems(): + if key.endswith('__gt'): + db_filters.add(getattr(self._db_cls, key[:-4]) > value) + elif key.endswith('__lt'): + db_filters.add(getattr(self._db_cls, key[:-4]) < value) + elif key.endswith('__ge'): + db_filters.add(getattr(self._db_cls, key[:-4]) >= value) + elif key.endswith('__le'): + db_filters.add(getattr(self._db_cls, key[:-4]) <= value) + elif key.endswith('__ne'): + db_filters.add(getattr(self._db_cls, key[:-4]) != value) + elif key.endswith('__in'): + db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) + else: + db_filters.add(getattr(self._db_cls, key) == value) + return self._db_cls.query.filter(*db_filters) def get(self, **filters): obj = self._get(**filters).first() if not obj: raise NotFound({'message': 'No %r (%r)' % (self._db_cls.__class__.__name__, filters)}) - if obj.user_id != self.user_id: + if getattr(obj, self._user_id_key) != self.user_id: raise Forbidden({'message': 'No authorized to access %r (%r)' % (self._db_cls.__class__.__name__, filters)}) return obj def create(self, **attrs): obj = self._db_cls(**attrs) - g.db.session.commit() + db.session.commit() return obj def read(self, **filters): return self._get(**filters) - def update(self, obj_id, **attrs): - obj = self.get(id=obj_id) - for key, values in attrs.iteritems(): - setattr(obj, key, values) - g.db.session.commit() - return obj + def update(self, filters, attrs): + result = self._get(**filters).update(attrs, synchronize_session=False) + db.session.commit() + return result def delete(self, obj_id): obj = self.get(id=obj_id) - g.db.session.delete(obj) - g.db.session.commit() + db.session.delete(obj) + db.session.commit() return obj diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index 5ada1a48..ce1c413f 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -1,6 +1,23 @@ +from datetime import datetime, timedelta from .abstract import AbstractController from pyaggr3g470r.models import Feed +DEFAULT_MAX_ERROR = 3 +DEFAULT_LIMIT = 5 + class FeedController(AbstractController): _db_cls = Feed + + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + from pyaggr3g470r.controllers import UserController + now = datetime.now() + user = UserController(self.user_id).get(id=self.user_id) + max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60) + feeds = [feed for feed in self.read(user_id=self.user_id, + error_count__le=max_error, + last_refreshed__lt=max_last_refresh).limit(limit)] + + self.update({'id__in': [feed.id for feed in feeds]}, + {'last_refreshed': now}) + return feeds diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py new file mode 100644 index 00000000..c6c1d545 --- /dev/null +++ b/pyaggr3g470r/controllers/user.py @@ -0,0 +1,7 @@ +from .abstract import AbstractController +from pyaggr3g470r.models import User + + +class UserController(AbstractController): + _db_cls = User + _user_id_key = 'id' diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index a116b9c3..edf560da 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -62,7 +62,7 @@ class PyAggResource(Resource): super(PyAggResource, self).__init__(*args, **kwargs) def get(self, obj_id=None): - return {'result': [self.controller.read(id=obj_id).dump()]} + return {'result': [self.controller.get(id=obj_id).dump()]} def put(self, obj_id=None): args = self.reqparse.parse_args() diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index 346898b5..94a5a433 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,7 +1,8 @@ from flask import g from flask.ext.restful import Resource, reqparse -from pyaggr3g470r.controllers import FeedController +from pyaggr3g470r.controllers.feed import FeedController, \ + DEFAULT_MAX_ERROR, DEFAULT_LIMIT from pyaggr3g470r.models import Feed from pyaggr3g470r.views.api.common import authenticate, to_response, \ @@ -72,7 +73,8 @@ class FeedAPI(PyAggResource): "Defines a RESTful API for Feed elements." controller_cls = FeedController editable_attrs = ['title', 'description', 'link', 'site_link', - 'email_notification', 'enabled'] + 'email_notification', 'enabled', 'last_refreshed', + 'last_error', 'error_count'] def __init__(self): self.reqparse = reqparse.RequestParser() @@ -87,5 +89,23 @@ class FeedAPI(PyAggResource): super(FeedAPI, self).__init__() +class FetchableFeedAPI(Resource): + + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.reqparse.add_argument('max_error', type=int, location='json', + default=DEFAULT_MAX_ERROR) + self.reqparse.add_argument('limit', type=int, location='json', + default=DEFAULT_LIMIT) + super(FetchableFeedAPI, self).__init__() + + def get(self): + args = self.reqparse.parse_args() + controller = FeedController(g.user.id) + return {'result': [feed.dump() for feed in controller.list_fetchable( + max_error=args['max_error'], limit=args['limit'])]} + + g.api.add_resource(FeedListAPI, '/feeds', endpoint='feeds.json') g.api.add_resource(FeedAPI, '/feeds/', endpoint='feed.json') +g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', endpoint='fetchable_feed.json') diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 6e467fb0..543f69b7 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -33,6 +33,7 @@ import random import hashlib import datetime from collections import namedtuple +from bootstrap import application as app, db from flask import render_template, request, flash, session, Response, \ url_for, redirect, g, current_app, make_response, jsonify from flask.ext.login import LoginManager, login_user, logout_user, \ @@ -55,17 +56,17 @@ if not conf.ON_HEROKU: import pyaggr3g470r.search as fastsearch -Principal(g.app) +Principal(app) # Create a permission with a single Need, in this case a RoleNeed. admin_permission = Permission(RoleNeed('admin')) login_manager = LoginManager() -login_manager.init_app(g.app) +login_manager.init_app(app) # # Management of the user's session. # -@identity_loaded.connect_via(g.app) +@identity_loaded.connect_via(app) def on_identity_loaded(sender, identity): # Set the identity user object identity.user = current_user @@ -80,13 +81,13 @@ def on_identity_loaded(sender, identity): for role in current_user.roles: identity.provides.add(RoleNeed(role.name)) -@g.app.before_request +@app.before_request def before_request(): g.user = current_user if g.user.is_authenticated(): g.user.last_seen = datetime.datetime.utcnow() - g.db.session.add(g.user) - g.db.session.commit() + db.session.add(g.user) + db.session.commit() @login_manager.user_loader def load_user(email): @@ -97,21 +98,21 @@ def load_user(email): # # Custom error pages. # -@g.app.errorhandler(401) +@app.errorhandler(401) def authentication_required(e): flash(gettext('Authentication required.'), 'info') return redirect(url_for('login')) -@g.app.errorhandler(403) +@app.errorhandler(403) def authentication_failed(e): flash(gettext('Forbidden.'), 'danger') return redirect(url_for('home')) -@g.app.errorhandler(404) +@app.errorhandler(404) def page_not_found(e): return render_template('errors/404.html'), 404 -@g.app.errorhandler(500) +@app.errorhandler(500) def internal_server_error(e): return render_template('errors/500.html'), 500 @@ -139,7 +140,7 @@ def get_timezone(): # # Views. # -@g.app.route('/login', methods=['GET', 'POST']) +@app.route('/login', methods=['GET', 'POST']) def login(): """ Log in view. @@ -160,7 +161,7 @@ def login(): return redirect(url_for('home')) return render_template('login.html', form=form) -@g.app.route('/api/csrf', methods=['GET']) +@app.route('/api/csrf', methods=['GET']) def get_csrf(): try: data = json.loads(request.data) @@ -185,7 +186,7 @@ def get_csrf(): return 'ok', 200 -@g.app.route('/logout') +@app.route('/logout') @login_required def logout(): """ @@ -206,7 +207,7 @@ def logout(): flash(gettext("Logged out successfully."), 'success') return redirect(url_for('login')) -@g.app.route('/signup', methods=['GET', 'POST']) +@app.route('/signup', methods=['GET', 'POST']) def signup(): """ Signup page. @@ -225,9 +226,9 @@ def signup(): email=form.email.data, pwdhash=generate_password_hash(form.password.data)) user.roles = [role_user] - g.db.session.add(user) + db.session.add(user) try: - g.db.session.commit() + db.session.commit() except IntegrityError: flash(gettext('Email already used.'), 'warning') return render_template('signup.html', form=form) @@ -244,7 +245,7 @@ def signup(): return render_template('signup.html', form=form) -@g.app.route('/') +@app.route('/') @login_required def home(): """ @@ -265,7 +266,7 @@ def home(): if limit != 'all': limit = int(limit) articles = articles.limit(limit) - unread = g.db.session.query(Article.feed_id, func.count(Article.id))\ + unread = db.session.query(Article.feed_id, func.count(Article.id))\ .filter(Article.readed == False, Article.user_id == g.user.id)\ .group_by(Article.feed_id).all() def gen_url(filter_=filter_, limit=limit, feed=feed_id): @@ -275,8 +276,8 @@ def home(): unread=dict(unread), articles=articles.all()) -@g.app.route('/fetch', methods=['GET']) -@g.app.route('/fetch/', methods=['GET']) +@app.route('/fetch', methods=['GET']) +@app.route('/fetch/', methods=['GET']) @login_required def fetch(feed_id=None): """ @@ -287,7 +288,7 @@ def fetch(feed_id=None): flash(gettext("Downloading articles..."), 'info') return redirect(redirect_url()) -@g.app.route('/about', methods=['GET']) +@app.route('/about', methods=['GET']) def about(): """ 'About' page. @@ -295,9 +296,9 @@ def about(): return render_template('about.html') -@g.app.route('/mark_as/', methods=['GET']) -@g.app.route('/mark_as//feed/', methods=['GET']) -@g.app.route('/mark_as//article/', methods=['GET']) +@app.route('/mark_as/', methods=['GET']) +@app.route('/mark_as//feed/', methods=['GET']) +@app.route('/mark_as//article/', methods=['GET']) @login_required @feed_access_required def mark_as(new_value='read', feed_id=None, article_id=None): @@ -316,12 +317,12 @@ def mark_as(new_value='read', feed_id=None, article_id=None): message = 'All article marked as %s.' articles.filter(Article.readed == (not readed)).update({"readed": readed}) flash(gettext(message % new_value), 'info') - g.db.session.commit() + db.session.commit() if readed: return redirect(redirect_url()) return redirect(url_for('home')) -@g.app.route('/like/', methods=['GET']) +@app.route('/like/', methods=['GET']) @login_required def like(article_id=None): """ @@ -331,10 +332,10 @@ def like(article_id=None): update({ "like": not Article.query.filter(Article.id == article_id).first().like }) - g.db.session.commit() + db.session.commit() return redirect(redirect_url()) -@g.app.route('/delete/', methods=['GET']) +@app.route('/delete/', methods=['GET']) @login_required def delete(article_id=None): """ @@ -342,8 +343,8 @@ def delete(article_id=None): """ article = Article.query.filter(Article.id == article_id).first() if article is not None and article.source.subscriber.id == g.user.id: - g.db.session.delete(article) - g.db.session.commit() + db.session.delete(article) + db.session.commit() try: fastsearch.delete_article(g.user.id, article.feed_id, article.id) except: @@ -355,7 +356,7 @@ def delete(article_id=None): return redirect(url_for('home')) -@g.app.route('/favorites', methods=['GET']) +@app.route('/favorites', methods=['GET']) @login_required def favorites(): """ @@ -370,8 +371,8 @@ def favorites(): nb_favorites += len(articles) return render_template('favorites.html', feeds=result, nb_favorites=nb_favorites) -@g.app.route('/unread/', methods=['GET']) -@g.app.route('/unread', methods=['GET']) +@app.route('/unread/', methods=['GET']) +@app.route('/unread', methods=['GET']) @login_required def unread(feed_id=None): """ @@ -389,7 +390,7 @@ def unread(feed_id=None): nb_unread += len(articles) return render_template('unread.html', feeds=result, nb_unread=nb_unread) -@g.app.route('/inactives', methods=['GET']) +@app.route('/inactives', methods=['GET']) @login_required def inactives(): """ @@ -409,7 +410,7 @@ def inactives(): inactives.append((feed, elapsed)) return render_template('inactives.html', inactives=inactives, nb_days=nb_days) -@g.app.route('/duplicates/', methods=['GET']) +@app.route('/duplicates/', methods=['GET']) @login_required def duplicates(feed_id=None): """ @@ -420,7 +421,7 @@ def duplicates(feed_id=None): duplicates = duplicate.compare_documents(feed) return render_template('duplicates.html', duplicates=duplicates, feed=feed) -@g.app.route('/index_database', methods=['GET']) +@app.route('/index_database', methods=['GET']) @login_required def index_database(): """ @@ -437,7 +438,7 @@ def index_database(): flash(gettext('Option not available on Heroku.'), 'success') return redirect(url_for('home')) -@g.app.route('/export', methods=['GET']) +@app.route('/export', methods=['GET']) @login_required def export_articles(): """ @@ -469,7 +470,7 @@ def export_articles(): return redirect(redirect_url()) return response -@g.app.route('/export_opml', methods=['GET']) +@app.route('/export_opml', methods=['GET']) @login_required def export_opml(): """ @@ -481,7 +482,7 @@ def export_opml(): response.headers['Content-Disposition'] = 'attachment; filename=feeds.opml' return response -@g.app.route('/search', methods=['GET']) +@app.route('/search', methods=['GET']) @login_required def search(): """ @@ -514,7 +515,7 @@ def search(): break return render_template('search.html', feeds=result, nb_articles=nb_articles, query=query) -@g.app.route('/management', methods=['GET', 'POST']) +@app.route('/management', methods=['GET', 'POST']) @login_required def management(): """ @@ -557,15 +558,15 @@ def management(): nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, not_on_heroku = not conf.ON_HEROKU) -@g.app.route('/history', methods=['GET']) +@app.route('/history', methods=['GET']) @login_required def history(): #user = User.query.filter(User.id == g.user.id).first() return render_template('history.html') -@g.app.route('/bookmarklet', methods=['GET']) -@g.app.route('/create_feed', methods=['GET', 'POST']) -@g.app.route('/edit_feed/', methods=['GET', 'POST']) +@app.route('/bookmarklet', methods=['GET']) +@app.route('/create_feed', methods=['GET', 'POST']) +@app.route('/edit_feed/', methods=['GET', 'POST']) @login_required @feed_access_required def edit_feed(feed_id=None): @@ -581,7 +582,7 @@ def edit_feed(feed_id=None): if feed_id is not None: # Edit an existing feed form.populate_obj(feed) - g.db.session.commit() + db.session.commit() flash(gettext('Feed successfully updated.'), 'success') return redirect('/edit_feed/' + str(feed_id)) else: @@ -593,7 +594,7 @@ def edit_feed(feed_id=None): enabled=form.enabled.data) g.user.feeds.append(new_feed) #user.feeds = sorted(user.feeds, key=lambda t: t.title.lower()) - g.db.session.commit() + db.session.commit() flash(gettext('Feed successfully created.'), 'success') utils.fetch(g.user.id, Feed.query.filter(Feed.link == form.link.data).first().id) @@ -615,7 +616,7 @@ def edit_feed(feed_id=None): existing_feed = [f for f in g.user.feeds if feed.link == request.args.get('url', None)] if len(existing_feed) == 0: g.user.feeds.append(Feed(link=request.args.get('url', None))) - g.db.session.commit() + db.session.commit() return jsonify({"message":"ok"}) return jsonify({"message":"Feed already in the database."}) @@ -623,7 +624,7 @@ def edit_feed(feed_id=None): return render_template('edit_feed.html', action=gettext("Add a feed"), form=form, \ not_on_heroku = not conf.ON_HEROKU) -@g.app.route('/delete_feed/', methods=['GET']) +@app.route('/delete_feed/', methods=['GET']) @login_required @feed_access_required def delete_feed(feed_id=None): @@ -631,12 +632,12 @@ def delete_feed(feed_id=None): Delete a feed with all associated articles. """ feed = Feed.query.filter(Feed.id == feed_id).first() - g.db.session.delete(feed) - g.db.session.commit() + db.session.delete(feed) + db.session.commit() flash(gettext('Feed') + ' ' + feed.title + ' ' + gettext('successfully deleted.'), 'success') return redirect(redirect_url()) -@g.app.route('/profile', methods=['GET', 'POST']) +@app.route('/profile', methods=['GET', 'POST']) @login_required def profile(): """ @@ -650,7 +651,7 @@ def profile(): form.populate_obj(user) if form.password.data != "": user.set_password(form.password.data) - g.db.session.commit() + db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') return redirect(url_for('profile')) else: @@ -660,7 +661,7 @@ def profile(): form = ProfileForm(obj=user) return render_template('profile.html', user=user, form=form) -@g.app.route('/delete_account', methods=['GET']) +@app.route('/delete_account', methods=['GET']) @login_required def delete_account(): """ @@ -668,14 +669,14 @@ def delete_account(): """ user = User.query.filter(User.email == g.user.email).first() if user is not None: - g.db.session.delete(user) - g.db.session.commit() + db.session.delete(user) + db.session.commit() flash(gettext('Your account has been deleted.'), 'success') else: flash(gettext('This user does not exist.'), 'danger') return redirect(url_for('login')) -@g.app.route('/expire_articles', methods=['GET']) +@app.route('/expire_articles', methods=['GET']) @login_required def expire_articles(): """ @@ -685,12 +686,12 @@ def expire_articles(): weeks_ago = current_time - datetime.timedelta(weeks=int(request.args.get('weeks', 10))) articles_to_delete = Article.query.filter(User.email == g.user.email, or_(Article.date < weeks_ago, Article.retrieved_date < weeks_ago)) for article in articles_to_delete: - g.db.session.delete(article) + db.session.delete(article) flash(gettext('Articles deleted.'), 'info') - g.db.session.commit() + db.session.commit() return redirect(redirect_url()) -@g.app.route('/confirm_account/', methods=['GET']) +@app.route('/confirm_account/', methods=['GET']) def confirm_account(activation_key=None): """ Confirm the account of a user. @@ -699,13 +700,13 @@ def confirm_account(activation_key=None): user = User.query.filter(User.activation_key == activation_key).first() if user is not None: user.activation_key = "" - g.db.session.commit() + db.session.commit() flash(gettext('Your account has been confirmed.'), 'success') else: flash(gettext('Impossible to confirm this account.'), 'danger') return redirect(url_for('login')) -@g.app.route('/recover', methods=['GET', 'POST']) +@app.route('/recover', methods=['GET', 'POST']) def recover(): """ Enables the user to recover its account when he has forgotten @@ -719,7 +720,7 @@ def recover(): characters = string.ascii_letters + string.digits password = "".join(random.choice(characters) for x in range(random.randint(8, 16))) user.set_password(password) - g.db.session.commit() + db.session.commit() # Send the confirmation email try: @@ -737,7 +738,7 @@ def recover(): # # Views dedicated to administration tasks. # -@g.app.route('/admin/dashboard', methods=['GET', 'POST']) +@app.route('/admin/dashboard', methods=['GET', 'POST']) @login_required @admin_permission.require(http_exception=403) def dashboard(): @@ -756,8 +757,8 @@ def dashboard(): users = User.query.all() return render_template('admin/dashboard.html', users=users, current_user=g.user, form=form) -@g.app.route('/admin/create_user', methods=['GET', 'POST']) -@g.app.route('/admin/edit_user/', methods=['GET', 'POST']) +@app.route('/admin/create_user', methods=['GET', 'POST']) +@app.route('/admin/edit_user/', methods=['GET', 'POST']) @login_required @admin_permission.require(http_exception=403) def create_user(user_id=None): @@ -775,7 +776,7 @@ def create_user(user_id=None): form.populate_obj(user) if form.password.data != "": user.set_password(form.password.data) - g.db.session.commit() + db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully updated.'), 'success') else: # Create a new user @@ -784,8 +785,8 @@ def create_user(user_id=None): pwdhash=generate_password_hash(form.password.data)) user.roles.extend([role_user]) user.activation_key = "" - g.db.session.add(user) - g.db.session.commit() + db.session.add(user) + db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully created.'), 'success') return redirect("/admin/edit_user/"+str(user.id)) else: @@ -801,7 +802,7 @@ def create_user(user_id=None): message = gettext('Add a new user') return render_template('/admin/create_user.html', form=form, message=message) -@g.app.route('/admin/user/', methods=['GET']) +@app.route('/admin/user/', methods=['GET']) @login_required @admin_permission.require(http_exception=403) def user(user_id=None): @@ -815,7 +816,7 @@ def user(user_id=None): flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) -@g.app.route('/admin/delete_user/', methods=['GET']) +@app.route('/admin/delete_user/', methods=['GET']) @login_required @admin_permission.require(http_exception=403) def delete_user(user_id=None): @@ -824,15 +825,15 @@ def delete_user(user_id=None): """ user = User.query.filter(User.id == user_id).first() if user is not None: - g.db.session.delete(user) - g.db.session.commit() + db.session.delete(user) + db.session.commit() flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully deleted.'), 'success') else: flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) -@g.app.route('/admin/enable_user/', methods=['GET']) -@g.app.route('/admin/disable_user/', methods=['GET']) +@app.route('/admin/enable_user/', methods=['GET']) +@app.route('/admin/disable_user/', methods=['GET']) @login_required @admin_permission.require() def disable_user(user_id=None): @@ -854,7 +855,7 @@ def disable_user(user_id=None): else: user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86] flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success') - g.db.session.commit() + db.session.commit() else: flash(gettext('This user does not exist.'), 'danger') return redirect(redirect_url()) -- cgit From 4f0ad9e442e64f69d420dea4d737805eefaaf981 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Wed, 21 Jan 2015 14:07:00 +0100 Subject: continuing refacto --- bootstrap.py | 3 +- .../versions/48f561c0ce6_add_column_entry_id.py | 2 +- migrations/versions/4b5c161e1ced_.py | 12 +- pyaggr3g470r/controllers/abstract.py | 2 +- pyaggr3g470r/controllers/article.py | 6 +- pyaggr3g470r/controllers/feed.py | 10 +- pyaggr3g470r/decorators.py | 5 +- pyaggr3g470r/emails.py | 1 + pyaggr3g470r/lib/client.py | 6 +- pyaggr3g470r/lib/crawler.py | 11 ++ pyaggr3g470r/models/__init__.py | 31 ++++-- pyaggr3g470r/views/api/article.py | 107 ++++-------------- pyaggr3g470r/views/api/common.py | 122 +++++++++++++++++---- pyaggr3g470r/views/api/feed.py | 120 ++++++-------------- pyaggr3g470r/views/views.py | 2 +- requirements.txt | 3 +- 16 files changed, 222 insertions(+), 221 deletions(-) create mode 100644 pyaggr3g470r/lib/crawler.py diff --git a/bootstrap.py b/bootstrap.py index bb1d1278..b9435d5c 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -27,8 +27,7 @@ from flask import Flask from flask.ext.sqlalchemy import SQLAlchemy # Create Flask application -application = Flask(__name__, static_folder='pyaggr3g470r/static', - template_folder='pyaggr3g470r/templates') +application = Flask('pyaggr3g470r') application.debug = conf.WEBSERVER_DEBUG # Create dummy secrey key so we can use sessions diff --git a/migrations/versions/48f561c0ce6_add_column_entry_id.py b/migrations/versions/48f561c0ce6_add_column_entry_id.py index 3f52a7a9..e5bc5735 100644 --- a/migrations/versions/48f561c0ce6_add_column_entry_id.py +++ b/migrations/versions/48f561c0ce6_add_column_entry_id.py @@ -1,7 +1,7 @@ """add column entry_id Revision ID: 48f561c0ce6 -Revises: +Revises: Create Date: 2015-02-18 21:17:19.346998 """ diff --git a/migrations/versions/4b5c161e1ced_.py b/migrations/versions/4b5c161e1ced_.py index e740e827..1efb5f81 100644 --- a/migrations/versions/4b5c161e1ced_.py +++ b/migrations/versions/4b5c161e1ced_.py @@ -1,4 +1,4 @@ -"""empty message +"""adding feed and user attributes for better feed retreiving Revision ID: 4b5c161e1ced Revises: None @@ -8,7 +8,7 @@ Create Date: 2015-01-17 01:04:10.187285 # revision identifiers, used by Alembic. revision = '4b5c161e1ced' -down_revision = None +down_revision = '48f561c0ce6' from alembic import op import sqlalchemy as sa @@ -18,15 +18,19 @@ def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('feed', sa.Column('error_count', sa.Integer(), nullable=True)) op.add_column('feed', sa.Column('last_error', sa.String(), nullable=True)) - op.add_column('feed', sa.Column('last_refreshed', sa.DateTime(), nullable=True)) + op.add_column('feed', sa.Column('last_modified', sa.DateTime(), nullable=True)) + op.add_column('feed', sa.Column('etag', sa.String(), nullable=True)) op.add_column('user', sa.Column('refresh_rate', sa.Integer(), nullable=True)) + op.add_column('article', sa.Column('guid', sa.String(), nullable=True)) ### end Alembic commands ### def downgrade(): ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'refresh_rate') - op.drop_column('feed', 'last_refreshed') + op.drop_column('feed', 'last_modified') op.drop_column('feed', 'last_error') op.drop_column('feed', 'error_count') + op.drop_column('feed', 'etag') + op.drop_column('article', 'guid') ### end Alembic commands ### diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py index 8960c3be..6fe45461 100644 --- a/pyaggr3g470r/controllers/abstract.py +++ b/pyaggr3g470r/controllers/abstract.py @@ -13,7 +13,7 @@ class AbstractController(object): if self.user_id: filters[self._user_id_key] = self.user_id db_filters = set() - for key, value in filters.iteritems(): + for key, value in filters.items(): if key.endswith('__gt'): db_filters.add(getattr(self._db_cls, key[:-4]) > value) elif key.endswith('__lt'): diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py index cfaf386d..0de223ee 100644 --- a/pyaggr3g470r/controllers/article.py +++ b/pyaggr3g470r/controllers/article.py @@ -6,10 +6,10 @@ from pyaggr3g470r.models import Article class ArticleController(AbstractController): _db_cls = Article - def read(self, obj_id): - article = super(ArticleController, self).read(obj_id) + def get(self, **filters): + article = super(ArticleController, self).read(**filters) if not article.readed: - self.update(obj_id, readed=True) + self.update(article.id, readed=True) return article def delete(self, obj_id): diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index ce1c413f..56cef997 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -16,8 +16,14 @@ class FeedController(AbstractController): max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60) feeds = [feed for feed in self.read(user_id=self.user_id, error_count__le=max_error, - last_refreshed__lt=max_last_refresh).limit(limit)] + last_modified=max_last_refresh).limit(limit)] self.update({'id__in': [feed.id for feed in feeds]}, - {'last_refreshed': now}) + {'last_modified': now}) return feeds + + def list_last_articles(self, feed_id, limit=50): + from pyaggr3g470r.controllers import ArticleController + return ArticleController(self.user_id)._get(feed_id=feed_id)\ + .order_by(ArticleController._db_cls.retrieved_date.desc())\ + .limit(limit) diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index a13dacac..9bae626d 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from threading import Thread from functools import wraps @@ -23,6 +23,7 @@ def async(f): thr.start() return wrapper + def feed_access_required(func): """ This decorator enables to check if a user has access to a feed. @@ -47,7 +48,7 @@ def handle_pyagg_error(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except PyAggError, error: + except PyAggError as error: flash(gettext(error.default_message), 'warning') return redirect(url_for('home')) return wrapper diff --git a/pyaggr3g470r/emails.py b/pyaggr3g470r/emails.py index 50f94761..9a129718 100644 --- a/pyaggr3g470r/emails.py +++ b/pyaggr3g470r/emails.py @@ -31,6 +31,7 @@ from pyaggr3g470r.decorators import async logger = logging.getLogger(__name__) + @async def send_async_email(mfrom, mto, msg): try: diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py index da6b1727..6b2fc9ae 100755 --- a/pyaggr3g470r/lib/client.py +++ b/pyaggr3g470r/lib/client.py @@ -1,16 +1,16 @@ #!/usr/bin/env python import json import requests -URL = 'domain.net' +import conf def get_client(email, password): client = requests.session() - client.get(URL + 'api/csrf', verify=False, + client.get(conf.PLATFORM_URL + 'api/csrf', verify=False, data=json.dumps({'email': email, 'password': password})) return client def get_articles(client): - return client.get(URL + 'api/v1.0/articles/').json + return client.get(conf.PLATFORM_URL + 'api/v1.0/articles/').json diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py new file mode 100644 index 00000000..1d7fca71 --- /dev/null +++ b/pyaggr3g470r/lib/crawler.py @@ -0,0 +1,11 @@ +import feedparser +import dateutil.parser.parse + + +def get_feed_content(feed): + etag = feed.get('etag', None) + last_modified = None + if feed.get('last_modified'): + last_modified = dateutil.parser.parse(feed['last_modified'])\ + .strftime('%a, %d %b %Y %H:%M:%S %Z') + return feedparser.parse(feed['link'], etag=etag, modified=last_modified) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py index f01eb3a3..27ee18b6 100644 --- a/pyaggr3g470r/models/__init__.py +++ b/pyaggr3g470r/models/__init__.py @@ -28,7 +28,8 @@ __license__ = "GPLv3" import re import json -import random, hashlib +import random +import hashlib from datetime import datetime from flask import g from sqlalchemy import asc, desc @@ -47,10 +48,8 @@ class User(db.Model, UserMixin): email = db.Column(db.String(254), index=True, unique=True) pwdhash = db.Column(db.String()) roles = db.relationship('Role', backref='user', lazy='dynamic') - activation_key = db.Column(db.String(128), default = - hashlib.sha512( - str(random.getrandbits(256)).encode("utf-8") - ).hexdigest()[:86]) + activation_key = db.Column(db.String(128), default=hashlib.sha512( + str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86]) date_created = db.Column(db.DateTime(), default=datetime.now) last_seen = db.Column(db.DateTime(), default=datetime.now) feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic', @@ -114,15 +113,21 @@ class Feed(db.Model): email_notification = db.Column(db.Boolean(), default=False) enabled = db.Column(db.Boolean(), default=True) created_date = db.Column(db.DateTime(), default=datetime.now) - last_refreshed = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # cache handling + etag = db.Column(db.String(), default="") + last_modified = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # error logging last_error = db.Column(db.String(), default="") error_count = db.Column(db.Integer(), default=0) + + # relationship + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) articles = db.relationship('Article', backref='source', lazy='dynamic', cascade='all,delete-orphan', order_by=desc("Article.date")) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - def __repr__(self): return '' % (self.title) @@ -132,14 +137,15 @@ class Feed(db.Model): "description": self.description, "link": self.link, "site_link": self.site_link, - "nb_articles": self.articles.count()} + "etag": self.etag, + "last_modified": self.last_modified} class Article(db.Model): """ Represent an article from a feed. """ - id = db.Column(db.Integer, primary_key = True) + id = db.Column(db.Integer, primary_key=True) entry_id = db.Column(db.String()) link = db.Column(db.String()) title = db.Column(db.String()) @@ -148,6 +154,7 @@ class Article(db.Model): like = db.Column(db.Boolean(), default=False) date = db.Column(db.DateTime(), default=datetime.now) retrieved_date = db.Column(db.DateTime(), default=datetime.now) + guid = db.Column(db.String(), default="") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) @@ -174,6 +181,7 @@ class Article(db.Model): "link": self.link, "content": self.content }) + def dump(self): return {"id": self.id, "title": self.title, @@ -184,5 +192,4 @@ class Article(db.Model): "date": self.date, "retrieved_date": self.retrieved_date, "feed_id": self.source.id, - "feed_name": self.source.title, - } + "feed_name": self.source.title} diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index 3642cda9..ebda6247 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -1,97 +1,36 @@ -import re -import dateutil.parser +from flask import g -from flask import request, g -from flask.ext.restful import Resource, reqparse - -from pyaggr3g470r.models import Article, Feed from pyaggr3g470r.controllers import ArticleController -from pyaggr3g470r.views.api.common import authenticate, to_response, \ - PyAggResource - +from pyaggr3g470r.views.api.common import PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti -class ArticleListAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate, to_response] - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('content', type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('date', type=str, location='json') - self.reqparse.add_argument('feed_id', type=int, location='json') - super(ArticleListAPI, self).__init__() +ARTICLE_ATTRS = {'title': {'type': str}, + 'content': {'type': str}, + 'link': {'type': str}, + 'date': {'type': str}, + 'feed_id': {'type': int}, + 'like': {'type': bool}, + 'readed': {'type': bool}} - def get(self): - """ - Returns a list of articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) +class ArticleNewAPI(PyAggResourceNew): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS - return {'result': [article.dump() for article in articles]} - def post(self): - """ - POST method - Create a new article. - """ - args = self.reqparse.parse_args() - article_dict = {} - for k, v in args.iteritems(): - if v != None: - article_dict[k] = v - else: - return {"message": "Missing argument: %s." % (k,)}, 400 - article_date = None - try: - article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) - except: - try: # trying to clean date field from letters - article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) - except: - return {"message": "Bad format for the date."}, 400 - article = Article(link=article_dict["link"], title=article_dict["title"], - content=article_dict["content"], readed=False, like=False, - date=article_date, user_id=g.user.id, - feed_id=article_dict["feed_id"]) - feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() - feed.articles.append(article) - try: - g.db.session.commit() - return {"message": "ok"}, 201 - except: - return {"message": "Impossible to create the article."}, 500 +class ArticleAPI(PyAggResourceExisting): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS -class ArticleAPI(PyAggResource): - "Defines a RESTful API for Article elements." - method_decorators = [authenticate, to_response] +class ArticlesAPI(PyAggResourceMulti): controller_cls = ArticleController - editable_attrs = ['like', 'readed'] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('like', type=bool, location='json') - self.reqparse.add_argument('readed', type=bool, location= 'json') - super(ArticleAPI, self).__init__() + attrs = ARTICLE_ATTRS -g.api.add_resource(ArticleListAPI, '/articles', endpoint='articles.json') -g.api.add_resource(ArticleAPI, '/articles/', - endpoint='article.json') +g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') +g.api.add_resource(ArticleAPI, '/article/', + endpoint='article.json') +g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index edf560da..c0759c03 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -1,6 +1,8 @@ +import json +import types from functools import wraps -from flask import request, g, session, Response, jsonify -from flask.ext.restful import Resource +from flask import request, g, session, Response +from flask.ext.restful import Resource, reqparse from pyaggr3g470r.models import User from pyaggr3g470r.lib.exceptions import PyAggError @@ -35,42 +37,122 @@ def authenticate(func): return wrapper +def default_handler(obj): + """JSON handler for default query formatting""" + if hasattr(obj, 'isoformat'): + return obj.isoformat() + if hasattr(obj, 'dump'): + return obj.dump() + if isinstance(obj, (set, frozenset, types.GeneratorType)): + return list(obj) + raise TypeError("Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj)) + + def to_response(func): def wrapper(*args, **kwargs): try: - res = func(*args, **kwargs) - except PyAggError, error: - response = jsonify(**error.message) + result = func(*args, **kwargs) + except PyAggError as error: + response = Response(json.dumps(result[0], default=default_handler)) response.status_code = error.status_code return response - if isinstance(res, tuple): - response = jsonify(**res[0]) - if len(res) > 1: - response.status_code = res[1] - return response - return res + status_code = 200 + if isinstance(result, tuple): + result, status_code = result + response = Response(json.dumps(result, default=default_handler), + status=status_code) + return response return wrapper -class PyAggResource(Resource): +class PyAggAbstractResource(Resource): method_decorators = [authenticate, to_response] - controller_cls = None - editable_attrs = [] def __init__(self, *args, **kwargs): self.controller = self.controller_cls(g.user.id) - super(PyAggResource, self).__init__(*args, **kwargs) + super(PyAggAbstractResource, self).__init__(*args, **kwargs) + + def reqparse_args(self, strict=False, default=True): + """ + strict: bool + if True will throw 400 error if args are defined and not in request + default: bool + if True, won't return defaults + + """ + parser = reqparse.RequestParser() + for attr_name, attrs in self.attrs.items(): + if not default and attr_name not in request.args: + continue + parser.add_argument(attr_name, location='json', **attrs) + return parser.parse_args(strict=strict) + + +class PyAggResourceNew(PyAggAbstractResource): + + def post(self): + return self.controller.create(**self.reqparse_args()), 201 + + +class PyAggResourceExisting(PyAggAbstractResource): def get(self, obj_id=None): - return {'result': [self.controller.get(id=obj_id).dump()]} + return self.controller.get(id=obj_id).dump() def put(self, obj_id=None): - args = self.reqparse.parse_args() + args = self.reqparse_args() new_values = {key: args[key] for key in - set(args).intersection(self.editable_attrs)} + set(args).intersection(self.attrs)} self.controller.update(obj_id, **new_values) - return {"message": "ok"} def delete(self, obj_id=None): self.controller.delete(obj_id) - return {"message": "ok"}, 204 + return None, 204 + + +class PyAggResourceMulti(PyAggAbstractResource): + + def get(self): + filters = self.reqparse_args(default=False) + return [res.dump() for res in self.controller.read(**filters).all()] + + def post(self): + status = 201 + results = [] + args = [] # FIXME + for arg in args: + try: + results.append(self.controller.create(**arg).id) + except Exception as error: + status = 206 + results.append(error) + return results, status + + def put(self): + status = 200 + results = [] + args = {} # FIXME + for obj_id, attrs in args.items(): + try: + new_values = {key: args[key] for key in + set(attrs).intersection(self.editable_attrs)} + self.controller.update(obj_id, **new_values) + results.append('ok') + except Exception as error: + status = 206 + results.append(error) + return results, status + + def delete(self): + status = 204 + results = [] + obj_ids = [] # FIXME extract some real ids + for obj_id in obj_ids: + try: + self.controller.delete(obj_id) + results.append('ok') + except Exception as error: + status = 206 + results.append(error) + return results, status diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index 94a5a433..e6f74cfd 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,92 +1,42 @@ +from datetime import datetime from flask import g from flask.ext.restful import Resource, reqparse -from pyaggr3g470r.controllers.feed import FeedController, \ - DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.models import Feed +from pyaggr3g470r.controllers.feed import FeedController, \ + DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.views.api.common import authenticate, to_response, \ - PyAggResource +from pyaggr3g470r.views.api.common import PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti -class FeedListAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate, to_response] +FEED_ATTRS = {'title': {'type': str}, + 'description': {'type': str}, + 'link': {'type': str}, + 'site_link': {'type': str}, + 'email_notification': {'type': bool, 'default': False}, + 'enabled': {'type': bool, 'default': True}, + 'etag': {'type': str, 'default': None}, + 'last_modified': {'type': datetime}, + 'last_error': {'type': datetime}, + 'error_count': {'type': int, 'default': 0}} - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', - type=unicode, default="", location='json') - self.reqparse.add_argument('description', - type=unicode, default="", location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', - type=unicode, default="", location='json') - self.reqparse.add_argument('email_notification', - type=bool, default=False, location='json') - self.reqparse.add_argument('enabled', - type=bool, default=True, location='json') - super(FeedListAPI, self).__init__() - def get(self): - """ - Returns a list of feeds. - """ - return {'result': [{"id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "email_notification": feed.email_notification, - "enabled": feed.enabled, - "created_date": feed.created_date, - } for feed in g.user.feeds]} - - def post(self): - """ - POST method - Create a new feed. - """ - args = self.reqparse.parse_args() - feed_dict = {} - for k, v in args.iteritems(): - if v != None: - feed_dict[k] = v - else: - return {'message': 'missing argument: %s' % (k,)}, 400 - new_feed = Feed(title=feed_dict["title"], - description=feed_dict["description"], - link=feed_dict["link"], - site_link=feed_dict["site_link"], - email_notification=feed_dict["email_notification"], - enabled=feed_dict["enabled"]) - g.user.feeds.append(new_feed) - try: - g.db.session.commit() - return {"message": "ok"} - except: - return {'message': 'Impossible to create the feed.'}, 500 - - -class FeedAPI(PyAggResource): - "Defines a RESTful API for Feed elements." +class FeedNewAPI(PyAggResourceNew): controller_cls = FeedController - editable_attrs = ['title', 'description', 'link', 'site_link', - 'email_notification', 'enabled', 'last_refreshed', - 'last_error', 'error_count'] + attrs = FEED_ATTRS - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type=unicode, location='json') - self.reqparse.add_argument('description', - type=unicode, location='json') - self.reqparse.add_argument('link', type=unicode, location='json') - self.reqparse.add_argument('site_link', type=unicode, location='json') - self.reqparse.add_argument('email_notification', - type=bool, location='json') - self.reqparse.add_argument('enabled', type=bool ,location='json') - super(FeedAPI, self).__init__() + +class FeedAPI(PyAggResourceExisting): + pass + controller_cls = FeedController + attrs = FEED_ATTRS + + +class FeedsAPI(PyAggResourceMulti): + pass + controller_cls = FeedController + attrs = FEED_ATTRS class FetchableFeedAPI(Resource): @@ -102,10 +52,12 @@ class FetchableFeedAPI(Resource): def get(self): args = self.reqparse.parse_args() controller = FeedController(g.user.id) - return {'result': [feed.dump() for feed in controller.list_fetchable( - max_error=args['max_error'], limit=args['limit'])]} + return [feed for feed in controller.list_fetchable( + max_error=args['max_error'], limit=args['limit'])] -g.api.add_resource(FeedListAPI, '/feeds', endpoint='feeds.json') -g.api.add_resource(FeedAPI, '/feeds/', endpoint='feed.json') -g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', endpoint='fetchable_feed.json') +g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') +g.api.add_resource(FeedAPI, '/feed/', endpoint='feed.json') +g.api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json') +g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', + endpoint='fetchable_feed.json') diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 543f69b7..053bb473 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -164,7 +164,7 @@ def login(): @app.route('/api/csrf', methods=['GET']) def get_csrf(): try: - data = json.loads(request.data) + data = json.loads(request.data.decode()) except ValueError: return Response(status=400) email = data.get('email') diff --git a/requirements.txt b/requirements.txt index b2f96657..60869a05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,10 +15,9 @@ Flask-RESTful Flask-Babel Flask-SSLify Flask-Migrate +flask-Script WTForms python-postmark whoosh python-dateutil alembic -flask-Migrate -flask-Script -- cgit From 5572851eca3b2f1bc56aed7232284acc436d2f49 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Sun, 1 Mar 2015 03:20:12 +0100 Subject: new crawler with cache control and error handling --- bootstrap.py | 2 +- conf.py | 1 + conf/conf.cfg-sample | 1 + migrations/versions/4b5c161e1ced_.py | 24 +++-- pyaggr3g470r/controllers/abstract.py | 15 ++- pyaggr3g470r/controllers/article.py | 11 +- pyaggr3g470r/controllers/feed.py | 17 ++- pyaggr3g470r/lib/crawler.py | 204 +++++++++++++++++++++++++++++++++-- pyaggr3g470r/lib/utils.py | 14 +++ pyaggr3g470r/models/__init__.py | 8 +- pyaggr3g470r/views/api/article.py | 36 +++++-- pyaggr3g470r/views/api/common.py | 79 +++++++------- pyaggr3g470r/views/api/feed.py | 37 +++---- pyaggr3g470r/views/views.py | 27 +---- 14 files changed, 348 insertions(+), 128 deletions(-) create mode 100644 pyaggr3g470r/lib/utils.py diff --git a/bootstrap.py b/bootstrap.py index b9435d5c..5d599146 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -12,7 +12,7 @@ if not (conf.WEBSERVER_DEBUG or conf.ON_HEROKU): gevent.monkey.patch_thread() -def set_logging(log_path, log_level=logging.INFO, +def set_logging(log_path, log_level=logging.DEBUG, log_format='%(asctime)s %(levelname)s %(message)s'): logger = logging.getLogger('pyaggr3g470r') formater = logging.Formatter(log_format) diff --git a/conf.py b/conf.py index 296f784d..6d963ebe 100644 --- a/conf.py +++ b/conf.py @@ -39,6 +39,7 @@ if not ON_HEROKU: RECAPTCHA_PRIVATE_KEY = config.get('misc', 'recaptcha_private_key') LOG_PATH = config.get('misc', 'log_path') PYTHON = config.get('misc', 'python') + NB_WORKER = config.getint('misc', 'nb_worker') WHOOSH_ENABLED = True diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index aab5ab5f..6e4bb7b7 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -5,6 +5,7 @@ recaptcha_public_key = recaptcha_private_key = log_path = ./pyaggr3g470r/var/pyaggr3g470r.log python = python3.3 +nb_worker = 5 [database] uri = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/aggregator [feedparser] diff --git a/migrations/versions/4b5c161e1ced_.py b/migrations/versions/4b5c161e1ced_.py index 1efb5f81..1fa91717 100644 --- a/migrations/versions/4b5c161e1ced_.py +++ b/migrations/versions/4b5c161e1ced_.py @@ -5,6 +5,7 @@ Revises: None Create Date: 2015-01-17 01:04:10.187285 """ +from datetime import datetime # revision identifiers, used by Alembic. revision = '4b5c161e1ced' @@ -15,22 +16,27 @@ import sqlalchemy as sa def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.add_column('feed', sa.Column('error_count', sa.Integer(), nullable=True)) + unix_start = datetime(1970, 1, 1) + # commands auto generated by Alembic - please adjust! ### + op.add_column('feed', sa.Column('error_count', sa.Integer(), nullable=True, + default=0, server_default="0")) op.add_column('feed', sa.Column('last_error', sa.String(), nullable=True)) - op.add_column('feed', sa.Column('last_modified', sa.DateTime(), nullable=True)) + op.add_column('feed', sa.Column('last_modified', sa.DateTime(), + nullable=True, default=unix_start, server_default=str(unix_start))) + op.add_column('feed', sa.Column('last_retreived', sa.DateTime(), + nullable=True, default=unix_start, server_default=str(unix_start))) op.add_column('feed', sa.Column('etag', sa.String(), nullable=True)) - op.add_column('user', sa.Column('refresh_rate', sa.Integer(), nullable=True)) - op.add_column('article', sa.Column('guid', sa.String(), nullable=True)) - ### end Alembic commands ### + op.add_column('user', sa.Column('refresh_rate', sa.Integer(), + nullable=True)) + # end Alembic commands ### def downgrade(): - ### commands auto generated by Alembic - please adjust! ### + # commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'refresh_rate') op.drop_column('feed', 'last_modified') op.drop_column('feed', 'last_error') op.drop_column('feed', 'error_count') + op.drop_column('feed', 'last_retreived') op.drop_column('feed', 'etag') - op.drop_column('article', 'guid') - ### end Alembic commands ### + # end Alembic commands ### diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py index 6fe45461..ebb73e30 100644 --- a/pyaggr3g470r/controllers/abstract.py +++ b/pyaggr3g470r/controllers/abstract.py @@ -1,5 +1,9 @@ +import logging from bootstrap import db -from pyaggr3g470r.lib.exceptions import Forbidden, NotFound +from sqlalchemy import update +from werkzeug.exceptions import Forbidden, NotFound + +logger = logging.getLogger(__name__) class AbstractController(object): @@ -9,7 +13,7 @@ class AbstractController(object): def __init__(self, user_id): self.user_id = user_id - def _get(self, **filters): + def _to_filters(self, **filters): if self.user_id: filters[self._user_id_key] = self.user_id db_filters = set() @@ -28,7 +32,10 @@ class AbstractController(object): db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) else: db_filters.add(getattr(self._db_cls, key) == value) - return self._db_cls.query.filter(*db_filters) + return db_filters + + def _get(self, **filters): + return self._db_cls.query.filter(*self._to_filters(**filters)) def get(self, **filters): obj = self._get(**filters).first() @@ -41,7 +48,9 @@ class AbstractController(object): return obj def create(self, **attrs): + attrs['user_id'] = self.user_id obj = self._db_cls(**attrs) + db.session.add(obj) db.session.commit() return obj diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py index 0de223ee..46ca0988 100644 --- a/pyaggr3g470r/controllers/article.py +++ b/pyaggr3g470r/controllers/article.py @@ -7,9 +7,9 @@ class ArticleController(AbstractController): _db_cls = Article def get(self, **filters): - article = super(ArticleController, self).read(**filters) + article = super(ArticleController, self).get(**filters) if not article.readed: - self.update(article.id, readed=True) + self.update({'id': article.id}, {'readed': True}) return article def delete(self, obj_id): @@ -18,3 +18,10 @@ class ArticleController(AbstractController): import pyaggr3g470r.search as fastsearch fastsearch.delete_article(self.user_id, obj.feed_id, obj_id) return obj + + def challenge(self, ids): + """Will return each id that wasn't found in the database.""" + for id_ in ids: + if self.read(**id_).first(): + continue + yield id_ diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index 56cef997..286bea1e 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -13,17 +13,12 @@ class FeedController(AbstractController): from pyaggr3g470r.controllers import UserController now = datetime.now() user = UserController(self.user_id).get(id=self.user_id) - max_last_refresh = now - timedelta(minutes=user.refresh_rate or 60) + max_last = now - timedelta(minutes=user.refresh_rate or 60) feeds = [feed for feed in self.read(user_id=self.user_id, - error_count__le=max_error, - last_modified=max_last_refresh).limit(limit)] + error_count__le=max_error, enabled=True, + last_retreived__lt=max_last).limit(limit)] - self.update({'id__in': [feed.id for feed in feeds]}, - {'last_modified': now}) + if feeds: + self.update({'id__in': [feed.id for feed in feeds]}, + {'last_retreived': now}) return feeds - - def list_last_articles(self, feed_id, limit=50): - from pyaggr3g470r.controllers import ArticleController - return ArticleController(self.user_id)._get(feed_id=feed_id)\ - .order_by(ArticleController._db_cls.retrieved_date.desc())\ - .limit(limit) diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 1d7fca71..6697e4c3 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -1,11 +1,199 @@ +import conf +import json +import logging +import requests import feedparser -import dateutil.parser.parse +import dateutil.parser +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor +from requests_futures.sessions import FuturesSession +from pyaggr3g470r.lib.utils import default_handler +logger = logging.getLogger(__name__) -def get_feed_content(feed): - etag = feed.get('etag', None) - last_modified = None - if feed.get('last_modified'): - last_modified = dateutil.parser.parse(feed['last_modified'])\ - .strftime('%a, %d %b %Y %H:%M:%S %Z') - return feedparser.parse(feed['link'], etag=etag, modified=last_modified) + +def extract_id(entry, keys=[('link', 'link'), + ('published', 'retrieved_date'), + ('updated', 'retrieved_date')], force_id=False): + entry_id = entry.get('entry_id') or entry.get('id') + if entry_id: + return {'entry_id': entry_id} + if not entry_id and force_id: + entry_id = hash("".join(entry[entry_key] for _, entry_key in keys + if entry_key in entry)) + else: + ids = {} + for entry_key, pyagg_key in keys: + if entry_key in entry and pyagg_key not in ids: + ids[pyagg_key] = entry[entry_key] + if 'date' in pyagg_key: + ids[pyagg_key] = dateutil.parser.parse(ids[pyagg_key])\ + .isoformat() + return ids + + +class AbstractCrawler: + __session__ = None + + def __init__(self, auth): + self.auth = auth + self.session = self.get_session() + self.url = conf.PLATFORM_URL + + @classmethod + def get_session(cls): + if cls.__session__ is None: + cls.__session__ = FuturesSession( + executor=ThreadPoolExecutor(max_workers=conf.NB_WORKER)) + cls.__session__.verify = False + return cls.__session__ + + def query_pyagg(self, method, urn, data=None): + if data is None: + data = {} + method = getattr(self.session, method) + return method("%sapi/v1.0/%s" % (self.url, urn), + auth=self.auth, data=json.dumps(data, + default=default_handler), + headers={'Content-Type': 'application/json'}) + + +class PyAggUpdater(AbstractCrawler): + + def __init__(self, feed, entries, headers, auth): + self.feed = feed + self.entries = entries + self.headers = headers + super(PyAggUpdater, self).__init__(auth) + + def to_article(self, entry): + date = datetime.now() + + for date_key in ('published', 'updated'): + if entry.get(date_key): + try: + date = dateutil.parser.parse(entry[date_key]) + except Exception: + pass + else: + break + content = '' + if entry.get('content'): + content = entry['content'][0]['value'] + elif entry.get('summary'): + content = entry['summary'] + + return {'feed_id': self.feed['id'], + 'entry_id': extract_id(entry).get('entry_id', None), + 'link': entry.get('link', self.feed['site_link']), + 'title': entry.get('title', 'No title'), + 'readed': False, 'like': False, + 'content': content, + 'retrieved_date': date.isoformat(), + 'date': date.isoformat()} + + def callback(self, response): + try: + results = response.result().json() + except Exception: + logger.exception('something went wront with feed %r %r %r %r', + self.feed, self.headers, response.result(), + getattr(response.result(), 'data', None)) + return + logger.debug('%r %r - %d entries were not matched', + self.feed['id'], self.feed['title'], len(results)) + for id_to_create in results: + entry = self.entries[tuple(sorted(id_to_create.items()))] + try: + logger.debug('creating %r - %r', entry['title'], id_to_create) + self.to_article(entry) + except: + logger.exception('%r %r %r something failed when parsing %r', + self.feed['title'], self.feed['id'], + self.feed['link'], entry) + self.query_pyagg('post', 'article', self.to_article(entry)) + + now = datetime.now() + logger.debug('%r %r - updating feed etag %r last_mod %r', + self.feed['id'], self.feed['title'], + self.headers.get('etag'), now) + + self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': 0, + 'etag': self.headers.get('etag', ''), + 'last_modified': self.headers.get('last-modified', '')}) + + +class FeedCrawler(AbstractCrawler): + + def __init__(self, feed, auth): + self.feed = feed + super(FeedCrawler, self).__init__(auth) + + def callback(self, response): + try: + response = response.result() + response.raise_for_status() + except Exception as error: + error_count = self.feed['error_count'] + 1 + logger.warn('%r %r - an error occured while fetching feed; bumping' + ' error count to %r', self.feed['title'], + self.feed['id'], error_count) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': error_count, + 'last_error': str(error)}) + return + + if response.status_code == 304: + logger.debug("%r %r - feed responded with 304", + self.feed['id'], self.feed['title']) + return + if self.feed['etag'] and response.headers.get('etag') \ + and response.headers.get('etag') == self.feed['etag']: + logger.debug("%r %r - feed responded with same etag (%d) %r", + self.feed['id'], self.feed['title'], + response.status_code, self.feed['link']) + return + ids, entries = [], {} + parsed_response = feedparser.parse(response.text) + for entry in parsed_response['entries']: + entries[tuple(sorted(extract_id(entry).items()))] = entry + ids.append(extract_id(entry)) + logger.debug('%r %r - found %d entries %r', + self.feed['id'], self.feed['title'], len(ids), ids) + future = self.query_pyagg('get', 'articles/challenge', {'ids': ids}) + updater = PyAggUpdater(self.feed, entries, response.headers, self.auth) + future.add_done_callback(updater.callback) + + +class CrawlerScheduler(AbstractCrawler): + + def __init__(self, username, password): + self.auth = (username, password) + super(CrawlerScheduler, self).__init__(self.auth) + + def prepare_headers(self, feed): + headers = {} + if feed.get('etag', None): + headers['If-None-Match'] = feed['etag'] + elif feed.get('last_modified'): + headers['If-Modified-Since'] = feed['last_modified'] + logger.debug('%r %r - calculated headers %r', + feed['id'], feed['title'], headers) + return headers + + def callback(self, response): + response = response.result() + response.raise_for_status() + feeds = response.json() + logger.debug('%d to fetch %r', len(feeds), feeds) + for feed in feeds: + logger.info('%r %r - fetching resources', + feed['id'], feed['title']) + future = self.session.get(feed['link'], + headers=self.prepare_headers(feed)) + future.add_done_callback(FeedCrawler(feed, self.auth).callback) + + def run(self): + logger.debug('retreving fetchable feed') + future = self.query_pyagg('get', 'feeds/fetchable') + future.add_done_callback(self.callback) diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py new file mode 100644 index 00000000..a4f4b3ec --- /dev/null +++ b/pyaggr3g470r/lib/utils.py @@ -0,0 +1,14 @@ +import types + +def default_handler(obj): + """JSON handler for default query formatting""" + if hasattr(obj, 'isoformat'): + return obj.isoformat() + if hasattr(obj, 'dump'): + return obj.dump() + if isinstance(obj, (set, frozenset, types.GeneratorType)): + return list(obj) + if isinstance(obj, BaseException): + return str(obj) + raise TypeError("Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj)) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py index 27ee18b6..25273036 100644 --- a/pyaggr3g470r/models/__init__.py +++ b/pyaggr3g470r/models/__init__.py @@ -116,7 +116,8 @@ class Feed(db.Model): # cache handling etag = db.Column(db.String(), default="") - last_modified = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + last_modified = db.Column(db.String(), default="") + last_retreived = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) # error logging last_error = db.Column(db.String(), default="") @@ -138,7 +139,9 @@ class Feed(db.Model): "link": self.link, "site_link": self.site_link, "etag": self.etag, - "last_modified": self.last_modified} + "error_count": self.error_count, + "last_modified": self.last_modified, + "last_retreived": self.last_retreived} class Article(db.Model): @@ -154,7 +157,6 @@ class Article(db.Model): like = db.Column(db.Boolean(), default=False) date = db.Column(db.DateTime(), default=datetime.now) retrieved_date = db.Column(db.DateTime(), default=datetime.now) - guid = db.Column(db.String(), default="") user_id = db.Column(db.Integer, db.ForeignKey('user.id')) feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index ebda6247..17881412 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -1,36 +1,58 @@ from flask import g +import dateutil.parser from pyaggr3g470r.controllers import ArticleController -from pyaggr3g470r.views.api.common import PyAggResourceNew, \ +from pyaggr3g470r.views.api.common import PyAggAbstractResource,\ + PyAggResourceNew, \ PyAggResourceExisting, \ PyAggResourceMulti -ARTICLE_ATTRS = {'title': {'type': str}, - 'content': {'type': str}, +ARTICLE_ATTRS = {'feed_id': {'type': str}, + 'entry_id': {'type': str}, 'link': {'type': str}, - 'date': {'type': str}, - 'feed_id': {'type': int}, - 'like': {'type': bool}, - 'readed': {'type': bool}} + 'title': {'type': str}, + 'readed': {'type': bool}, 'like': {'type': bool}, + 'content': {'type': str}, + 'date': {'type': str}, 'retrieved_date': {'type': str}} class ArticleNewAPI(PyAggResourceNew): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] class ArticleAPI(PyAggResourceExisting): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] class ArticlesAPI(PyAggResourceMulti): controller_cls = ArticleController attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] + + +class ArticlesChallenge(PyAggAbstractResource): + controller_cls = ArticleController + attrs = {'ids': {'type': list, 'default': []}} + to_date = ['date', 'retrieved_date'] + + def get(self): + parsed_args = self.reqparse_args() + for id_dict in parsed_args['ids']: + for key in self.to_date: + if key in id_dict: + id_dict[key] = dateutil.parser.parse(id_dict[key]) + + return self.controller.challenge(parsed_args['ids']) g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') g.api.add_resource(ArticleAPI, '/article/', endpoint='article.json') g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') +g.api.add_resource(ArticlesChallenge, '/articles/challenge', + endpoint='articles_challenge.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index c0759c03..a9d35411 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -1,12 +1,16 @@ import json -import types +import logging +import dateutil.parser from functools import wraps from flask import request, g, session, Response from flask.ext.restful import Resource, reqparse +from pyaggr3g470r.lib.utils import default_handler from pyaggr3g470r.models import User from pyaggr3g470r.lib.exceptions import PyAggError +logger = logging.getLogger(__name__) + def authenticate(func): """ @@ -24,55 +28,47 @@ def authenticate(func): # authentication via HTTP only auth = request.authorization try: - email = auth.username - user = User.query.filter(User.email == email).first() - if user and user.check_password(auth.password) and user.activation_key == "": + user = User.query.filter(User.nickname == auth.username).first() + if user and user.check_password(auth.password) \ + and user.activation_key == "": g.user = user - return func(*args, **kwargs) - except AttributeError: - pass - - return Response('', 401, - {'WWWAuthenticate':'Basic realm="Login Required"'}) + except Exception: + return Response('', 401, + {'WWWAuthenticate': + 'Basic realm="Login Required"'}) + return func(*args, **kwargs) return wrapper -def default_handler(obj): - """JSON handler for default query formatting""" - if hasattr(obj, 'isoformat'): - return obj.isoformat() - if hasattr(obj, 'dump'): - return obj.dump() - if isinstance(obj, (set, frozenset, types.GeneratorType)): - return list(obj) - raise TypeError("Object of type %s with value of %r " - "is not JSON serializable" % (type(obj), obj)) - - def to_response(func): def wrapper(*args, **kwargs): + status_code = 200 try: result = func(*args, **kwargs) except PyAggError as error: - response = Response(json.dumps(result[0], default=default_handler)) - response.status_code = error.status_code - return response - status_code = 200 - if isinstance(result, tuple): - result, status_code = result - response = Response(json.dumps(result, default=default_handler), + return Response(json.dumps(error, default=default_handler), status=status_code) - return response + if isinstance(result, Response): + return result + elif isinstance(result, tuple): + result, status_code = result + return Response(json.dumps(result, default=default_handler), + status=status_code) return wrapper class PyAggAbstractResource(Resource): method_decorators = [authenticate, to_response] + attrs = {} + to_date = [] def __init__(self, *args, **kwargs): - self.controller = self.controller_cls(g.user.id) super(PyAggAbstractResource, self).__init__(*args, **kwargs) + @property + def controller(self): + return self.controller_cls(getattr(g.user, 'id', None)) + def reqparse_args(self, strict=False, default=True): """ strict: bool @@ -83,10 +79,17 @@ class PyAggAbstractResource(Resource): """ parser = reqparse.RequestParser() for attr_name, attrs in self.attrs.items(): - if not default and attr_name not in request.args: + if not default and attr_name not in request.json: continue parser.add_argument(attr_name, location='json', **attrs) - return parser.parse_args(strict=strict) + parsed = parser.parse_args(strict=strict) + for field in self.to_date: + if parsed.get(field): + try: + parsed[field] = dateutil.parser.parse(parsed[field]) + except Exception: + logger.exception('failed to parse %r', parsed[field]) + return parsed class PyAggResourceNew(PyAggAbstractResource): @@ -98,13 +101,13 @@ class PyAggResourceNew(PyAggAbstractResource): class PyAggResourceExisting(PyAggAbstractResource): def get(self, obj_id=None): - return self.controller.get(id=obj_id).dump() + return self.controller.get(id=obj_id) def put(self, obj_id=None): - args = self.reqparse_args() + args = self.reqparse_args(default=False) new_values = {key: args[key] for key in set(args).intersection(self.attrs)} - self.controller.update(obj_id, **new_values) + self.controller.update({'id': obj_id}, new_values) def delete(self, obj_id=None): self.controller.delete(obj_id) @@ -115,7 +118,7 @@ class PyAggResourceMulti(PyAggAbstractResource): def get(self): filters = self.reqparse_args(default=False) - return [res.dump() for res in self.controller.read(**filters).all()] + return [res for res in self.controller.read(**filters).all()] def post(self): status = 201 @@ -137,7 +140,7 @@ class PyAggResourceMulti(PyAggAbstractResource): try: new_values = {key: args[key] for key in set(attrs).intersection(self.editable_attrs)} - self.controller.update(obj_id, **new_values) + self.controller.update({'id': obj_id}, new_values) results.append('ok') except Exception as error: status = 206 diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index e6f74cfd..625ad52d 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -1,11 +1,10 @@ -from datetime import datetime from flask import g -from flask.ext.restful import Resource, reqparse from pyaggr3g470r.controllers.feed import FeedController, \ DEFAULT_MAX_ERROR, DEFAULT_LIMIT -from pyaggr3g470r.views.api.common import PyAggResourceNew, \ +from pyaggr3g470r.views.api.common import PyAggAbstractResource, \ + PyAggResourceNew, \ PyAggResourceExisting, \ PyAggResourceMulti @@ -16,44 +15,40 @@ FEED_ATTRS = {'title': {'type': str}, 'site_link': {'type': str}, 'email_notification': {'type': bool, 'default': False}, 'enabled': {'type': bool, 'default': True}, - 'etag': {'type': str, 'default': None}, - 'last_modified': {'type': datetime}, - 'last_error': {'type': datetime}, + 'etag': {'type': str, 'default': ''}, + 'last_modified': {'type': str}, + 'last_retreived': {'type': str}, + 'last_error': {'type': str}, 'error_count': {'type': int, 'default': 0}} class FeedNewAPI(PyAggResourceNew): controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] class FeedAPI(PyAggResourceExisting): - pass controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] class FeedsAPI(PyAggResourceMulti): - pass controller_cls = FeedController attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] -class FetchableFeedAPI(Resource): - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('max_error', type=int, location='json', - default=DEFAULT_MAX_ERROR) - self.reqparse.add_argument('limit', type=int, location='json', - default=DEFAULT_LIMIT) - super(FetchableFeedAPI, self).__init__() +class FetchableFeedAPI(PyAggAbstractResource): + controller_cls = FeedController + to_date = ['date', 'last_retreived'] + attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR}, + 'limit': {'type': int, 'default': DEFAULT_LIMIT}} def get(self): - args = self.reqparse.parse_args() - controller = FeedController(g.user.id) - return [feed for feed in controller.list_fetchable( - max_error=args['max_error'], limit=args['limit'])] + return [feed for feed in self.controller.list_fetchable( + **self.reqparse_args())] g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 053bb473..66170a77 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -156,35 +156,12 @@ def login(): login_user(user) g.user = user session['email'] = form.email.data - identity_changed.send(current_app._get_current_object(), identity=Identity(user.id)) + identity_changed.send(current_app._get_current_object(), + identity=Identity(user.id)) flash(gettext("Logged in successfully."), 'success') return redirect(url_for('home')) return render_template('login.html', form=form) -@app.route('/api/csrf', methods=['GET']) -def get_csrf(): - try: - data = json.loads(request.data.decode()) - except ValueError: - return Response(status=400) - email = data.get('email') - password = data.get('password') - if login is None or password is None: - return Response(status=401) - user = User.query.filter(User.email == email).first() - if not user: - return Response(status=404) - if not user.check_password(password): - return Response(status=401) - if not user.activation_key == "": - return Response(status=403) - login_user(user) - g.user = user - session['email'] = email - identity_changed.send(current_app._get_current_object(), - identity=Identity(user.id)) - return 'ok', 200 - @app.route('/logout') @login_required -- cgit From a4fb151ea53d8054cc8e3fb309395c8fa0e23aaf Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Sun, 1 Mar 2015 14:08:02 +0100 Subject: fixing/restoring logging level --- bootstrap.py | 4 +-- documentation/deployment.rst | 3 +- documentation/web-services.rst | 72 +++++++++++++++++++------------------- pyaggr3g470r/decorators.py | 13 ------- pyaggr3g470r/lib/client.py | 16 --------- pyaggr3g470r/lib/crawler.py | 7 ++-- pyaggr3g470r/lib/exceptions.py | 13 ------- pyaggr3g470r/search.py | 2 +- pyaggr3g470r/static/js/articles.js | 8 +++-- pyaggr3g470r/templates/layout.html | 10 +++--- pyaggr3g470r/views/api/common.py | 33 ++++++++--------- pyaggr3g470r/views/article.py | 2 +- pyaggr3g470r/views/feed.py | 6 ++-- requirements.txt | 2 +- runserver.py | 2 +- 15 files changed, 75 insertions(+), 118 deletions(-) delete mode 100755 pyaggr3g470r/lib/client.py delete mode 100644 pyaggr3g470r/lib/exceptions.py diff --git a/bootstrap.py b/bootstrap.py index 5d599146..83862640 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -21,14 +21,14 @@ def set_logging(log_path, log_level=logging.DEBUG, logger.addHandler(handler) logger.setLevel(log_level) -set_logging(conf.LOG_PATH) - from flask import Flask from flask.ext.sqlalchemy import SQLAlchemy # Create Flask application application = Flask('pyaggr3g470r') application.debug = conf.WEBSERVER_DEBUG +set_logging(conf.LOG_PATH, log_level=logging.DEBUG if conf.WEBSERVER_DEBUG + else logging.INFO) # Create dummy secrey key so we can use sessions application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) diff --git a/documentation/deployment.rst b/documentation/deployment.rst index d0639c45..d06d55fe 100644 --- a/documentation/deployment.rst +++ b/documentation/deployment.rst @@ -92,6 +92,7 @@ If you want to use PostgreSQL .. code-block:: bash $ sudo apt-get install postgresql postgresql-server-dev-9.3 postgresql-client + $ pip install psycopg2 $ echo "127.0.0.1:5432:aggregator:pgsqluser:pgsqlpwd" > ~/.pgpass $ chmod 700 ~/.pgpass $ sudo -u postgres createuser pgsqluser --no-superuser --createdb --no-createrole @@ -131,7 +132,7 @@ Configuration ============= Configuration (database url, email, proxy, user agent, etc.) is done via the file *conf/conf.cfg*. -Check these configuration before executing *db_create.py*. +Check these configuration before executing *db_create.py*. If you want to use pyAggr3g470r with Tor/Privoxy, you just have to set the value of *http_proxy* (most of the time: *http_proxy = 127.0.0.1:8118**). Else leave the value blank. diff --git a/documentation/web-services.rst b/documentation/web-services.rst index 2d724569..dd2d0125 100644 --- a/documentation/web-services.rst +++ b/documentation/web-services.rst @@ -7,10 +7,11 @@ Articles .. code-block:: python >>> import requests, json - >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", auth=("your-email", "your-password")) - >>> r.status_code-block - 200 - >>> rjson = json.loads(r.text) + >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... auth=("your-nickname", "your-password")) + >>> r.status_code + 200 # OK + >>> rjson = r.json() >>> rjson["result"][0]["title"] u'Sponsors required for KDE code sprint in Randa' >>> rjson["result"][0]["date"] @@ -20,15 +21,19 @@ Possible parameters: .. code-block:: bash - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=unread&feed=24" - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=read&feed=24&limit=20" - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=all&feed=24&limit=20" + $ curl --user your-nickname:your-password "https://pyaggr3g470r.herokuapp.com/api/v2.0/articles" -H 'Content-Type: application/json' --data='{"feed": 24}' -Get an article: +Get an article with another way to pass credentials : .. code-block:: bash - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84566" + $ curl "https://your-nickname:your-password@pyaggr3g470r.herokuapp.com/api/v2.0/article/84566" + +And delete it : + +.. code-block:: bash + + $ curl -XDELETE "https://your-nickname:your-password@pyaggr3g470r.herokuapp.com/api/v2.0/article/84566" Add an article: @@ -36,13 +41,18 @@ Add an article: >>> import requests, json >>> headers = {'Content-type': 'application/json', 'Accept': 'application/json'} - >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/', 'title': 'Sortie de pyAggr3g470r 5.3', 'content':'La page principale de pyAggr3g470r a été améliorée...', 'date':'06/23/2014 11:42 AM', 'feed_id':'42'} - >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) - >>> print r.content - { - "message": "ok" - } - >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?feed=42&limit=1", auth=("your-email", "your-password")) + >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/', + ... 'title': 'Sortie de pyAggr3g470r 5.3', + ... 'content':'La page principale de pyAggr3g470r a été améliorée...', + ... 'date':'2014/06/23T11:42:20 GMT', + ... 'feed_id':'42'} + >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/article", + ... headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload)) + >>> print r.status_code + 201 # Created + >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... auth=("your-nickname", "your-password") + ... data=json.dumps({'feed_id': 42, 'limit': 1})) >>> print json.loads(r.content)["result"][0]["title"] Sortie de pyAggr3g470r 5.3 @@ -51,30 +61,20 @@ Update an article: .. code-block:: python >>> payload = {"like":True, "readed":False} - >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/65", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) - >>> print r.content - { - "message": "ok" - } + >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/65", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload)) + >>> print r.status_code + 200 # OK Delete an article: .. code-block:: python - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password")) + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", auth=("your-nickname", "your-password")) >>> print r.status_code - 200 - >>> print r.content - { - "message": "ok" - } - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password")) + 204 # deleted - No content + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", auth=("your-nickname", "your-password")) >>> print r.status_code - 200 - >>> print r.content - { - "message": "Article not found." - } + 404 # not found Feeds ----- @@ -84,17 +84,17 @@ Add a feed: .. code-block:: python >>> payload = {'link': 'http://blog.cedricbonhomme.org/feed'} - >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) + >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload)) Update a feed: .. code-block:: python >>> payload = {"title":"Feed new title", "description":"New description"} - >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/42", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) + >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/42", headers=headers, auth=("your-nickname", "your-password"), data=json.dumps(payload)) Delete a feed: .. code-block:: python - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/29", auth=("your-email", "your-password")) + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/29", auth=("your-nickname", "your-password")) diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index 9bae626d..9e8f9c0b 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -9,7 +9,6 @@ from flask.ext.babel import gettext from flask.ext.login import login_required from pyaggr3g470r.models import Feed -from pyaggr3g470r.lib.exceptions import PyAggError def async(f): @@ -43,20 +42,8 @@ def feed_access_required(func): return decorated -def handle_pyagg_error(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except PyAggError as error: - flash(gettext(error.default_message), 'warning') - return redirect(url_for('home')) - return wrapper - - def pyagg_default_decorator(func): @login_required - @handle_pyagg_error @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) diff --git a/pyaggr3g470r/lib/client.py b/pyaggr3g470r/lib/client.py deleted file mode 100755 index 6b2fc9ae..00000000 --- a/pyaggr3g470r/lib/client.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import json -import requests -import conf - - -def get_client(email, password): - client = requests.session() - client.get(conf.PLATFORM_URL + 'api/csrf', verify=False, - data=json.dumps({'email': email, - 'password': password})) - return client - - -def get_articles(client): - return client.get(conf.PLATFORM_URL + 'api/v1.0/articles/').json diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 6697e4c3..de770934 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -10,6 +10,7 @@ from requests_futures.sessions import FuturesSession from pyaggr3g470r.lib.utils import default_handler logger = logging.getLogger(__name__) +API_ROOT = "api/v2.0/" def extract_id(entry, keys=[('link', 'link'), @@ -52,7 +53,7 @@ class AbstractCrawler: if data is None: data = {} method = getattr(self.session, method) - return method("%sapi/v1.0/%s" % (self.url, urn), + return method("%s%s%s" % (self.url, API_ROOT, urn), auth=self.auth, data=json.dumps(data, default=default_handler), headers={'Content-Type': 'application/json'}) @@ -193,7 +194,7 @@ class CrawlerScheduler(AbstractCrawler): headers=self.prepare_headers(feed)) future.add_done_callback(FeedCrawler(feed, self.auth).callback) - def run(self): + def run(self, **kwargs): logger.debug('retreving fetchable feed') - future = self.query_pyagg('get', 'feeds/fetchable') + future = self.query_pyagg('get', 'feeds/fetchable', kwargs) future.add_done_callback(self.callback) diff --git a/pyaggr3g470r/lib/exceptions.py b/pyaggr3g470r/lib/exceptions.py deleted file mode 100644 index 30c71a5c..00000000 --- a/pyaggr3g470r/lib/exceptions.py +++ /dev/null @@ -1,13 +0,0 @@ -class PyAggError(Exception): - status_code = None - default_message = '' - - -class Forbidden(PyAggError): - status_code = 403 - default_message = 'You do not have the rights to access that resource' - - -class NotFound(PyAggError): - status_code = 404 - default_message = 'Resource was not found' diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py index 89fa0860..a7f780df 100644 --- a/pyaggr3g470r/search.py +++ b/pyaggr3g470r/search.py @@ -102,7 +102,7 @@ def delete_article(user_id, feed_id, article_id): try: ix = open_dir(indexdir) except (EmptyIndexError, OSError): - raise EmptyIndexError + return writer = ix.writer() document = And([Term("user_id", user_id), Term("feed_id", feed_id), Term("article_id", article_id)]) diff --git a/pyaggr3g470r/static/js/articles.js b/pyaggr3g470r/static/js/articles.js index 51273f3d..312a5cb6 100644 --- a/pyaggr3g470r/static/js/articles.js +++ b/pyaggr3g470r/static/js/articles.js @@ -18,6 +18,8 @@ * along with this program. If not, see . */ +API_ROOT = 'api/v2.0/' + if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } +function ($) { @@ -78,7 +80,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // Encode your data as JSON. data: data, // This is the type of data you're expecting back from the server. - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, @@ -114,7 +116,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // Encode your data as JSON. data: data, // This is the type of data you're expecting back from the server. - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, @@ -132,7 +134,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // sends the updates to the server $.ajax({ type: 'DELETE', - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html index 4dc62350..6b929bf3 100644 --- a/pyaggr3g470r/templates/layout.html +++ b/pyaggr3g470r/templates/layout.html @@ -7,9 +7,9 @@ {% if head_title %}{{ head_title }} - {% endif %}pyAggr3g470r - + - +