diff options
47 files changed, 1590 insertions, 839 deletions
@@ -21,3 +21,4 @@ pyaggr3g470r/var/indexdir/ # Virtualenv venv +build diff --git a/bootstrap.py b/bootstrap.py index 85bac49b..5cfd2250 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,5 +1,6 @@ # required imports and code exection for basic functionning +import os import conf import logging @@ -12,4 +13,28 @@ def set_logging(log_path, log_level=logging.INFO, logger.addHandler(handler) logger.setLevel(log_level) -set_logging(conf.LOG_PATH) +from flask import Flask +from flask.ext.sqlalchemy import SQLAlchemy + +# Create Flask application +application = Flask('pyaggr3g470r') +application.debug = conf.WEBSERVER_DEBUG +set_logging(conf.LOG_PATH, log_level=logging.DEBUG if conf.WEBSERVER_DEBUG + else logging.INFO) + +# Create dummy secrey key so we can use sessions +application.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) +if not application.config['SECRET_KEY']: + application.config['SECRET_KEY'] = os.urandom(12) +application.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI + +application.config['RECAPTCHA_USE_SSL'] = True +application.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY +application.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY + +db = SQLAlchemy(application) + +def populate_g(): + from flask import g + g.db = db + g.app = application @@ -39,6 +39,7 @@ if not ON_HEROKU: RECAPTCHA_PRIVATE_KEY = config.get('misc', 'recaptcha_private_key') LOG_PATH = config.get('misc', 'log_path') PYTHON = config.get('misc', 'python') + NB_WORKER = config.getint('misc', 'nb_worker') WHOOSH_ENABLED = True diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index 813b2ac2..4f1e06c6 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -5,6 +5,7 @@ recaptcha_public_key = recaptcha_private_key = log_path = ./pyaggr3g470r/var/pyaggr3g470r.log python = python3.3 +nb_worker = 5 [database] uri = postgres://pgsqluser:pgsqlpwd@127.0.0.1:5432/aggregator [feedparser] diff --git a/db_create.py b/db_create.py deleted file mode 100755 index 8e743515..00000000 --- a/db_create.py +++ /dev/null @@ -1,80 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.3 $" -__date__ = "$Date: 2014/03/16 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "AGPLv3" - -import os -import bootstrap - -from pyaggr3g470r import db -from pyaggr3g470r.models import User, Role -from werkzeug import generate_password_hash - -from sqlalchemy.engine import reflection -from sqlalchemy.schema import ( - MetaData, - Table, - DropTable, - ForeignKeyConstraint, - DropConstraint, - ) - - -def db_DropEverything(db): - # From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything - - conn = db.engine.connect() - - # the transaction only applies if the DB supports - # transactional DDL, i.e. Postgresql, MS SQL Server - trans = conn.begin() - - inspector = reflection.Inspector.from_engine(db.engine) - - # gather all data first before dropping anything. - # some DBs lock after things have been dropped in - # a transaction. - metadata = MetaData() - - tbs = [] - all_fks = [] - - for table_name in inspector.get_table_names(): - fks = [] - for fk in inspector.get_foreign_keys(table_name): - if not fk['name']: - continue - fks.append( - ForeignKeyConstraint((), (), name=fk['name']) - ) - t = Table(table_name, metadata, *fks) - tbs.append(t) - all_fks.extend(fks) - - for fkc in all_fks: - conn.execute(DropConstraint(fkc)) - - for table in tbs: - conn.execute(DropTable(table)) - - trans.commit() - -db_DropEverything(db) -db.create_all() - -role_admin = Role(name="admin") -role_user = Role(name="user") - -user1 = User(nickname="admin", - email=os.environ.get("ADMIN_EMAIL", "root@pyAggr3g470r.localhost"), - pwdhash=generate_password_hash(os.environ.get("ADMIN_PASSWORD", "password")), - activation_key="") -user1.roles.extend([role_admin, role_user]) - -db.session.add(user1) -db.session.commit() diff --git a/documentation/deployment.rst b/documentation/deployment.rst index d0639c45..d06d55fe 100644 --- a/documentation/deployment.rst +++ b/documentation/deployment.rst @@ -92,6 +92,7 @@ If you want to use PostgreSQL .. code-block:: bash $ sudo apt-get install postgresql postgresql-server-dev-9.3 postgresql-client + $ pip install psycopg2 $ echo "127.0.0.1:5432:aggregator:pgsqluser:pgsqlpwd" > ~/.pgpass $ chmod 700 ~/.pgpass $ sudo -u postgres createuser pgsqluser --no-superuser --createdb --no-createrole @@ -131,7 +132,7 @@ Configuration ============= Configuration (database url, email, proxy, user agent, etc.) is done via the file *conf/conf.cfg*. -Check these configuration before executing *db_create.py*. +Check these configuration before executing *db_create.py*. If you want to use pyAggr3g470r with Tor/Privoxy, you just have to set the value of *http_proxy* (most of the time: *http_proxy = 127.0.0.1:8118**). Else leave the value blank. diff --git a/documentation/web-services.rst b/documentation/web-services.rst index 2d724569..16a425ad 100644 --- a/documentation/web-services.rst +++ b/documentation/web-services.rst @@ -7,28 +7,23 @@ Articles .. code-block:: python >>> import requests, json - >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", auth=("your-email", "your-password")) - >>> r.status_code-block - 200 - >>> rjson = json.loads(r.text) - >>> rjson["result"][0]["title"] - u'Sponsors required for KDE code sprint in Randa' - >>> rjson["result"][0]["date"] - u'Wed, 18 Jun 2014 14:25:18 GMT' - -Possible parameters: - -.. code-block:: bash - - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=unread&feed=24" - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=read&feed=24&limit=20" - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?filter_=all&feed=24&limit=20" - -Get an article: - -.. code-block:: bash - - $ curl --user your-email:your-password "https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84566" + >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/1s", + ... headers={'Content-type': 'application/json'}, + ... auth=("your-nickname", "your-password")) + >>> r.status_code + 200 # OK + >>> rjson = r.json() + >>> rjson["title"] + 'Sponsors required for KDE code sprint in Randa' + >>> rjson["date"] + 'Wed, 18 Jun 2014 14:25:18 GMT' + >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/1s", + ... headers={'Content-type': 'application/json'}, + ... auth=("your-nickname", "your-password"), + ... data=json.dumps({'id__in': [1, 2]})) + >>> r.json() + [{'id': 1, 'title': 'article1', [...]}, + {'id': 2, 'title': 'article2', [...]}] Add an article: @@ -36,45 +31,85 @@ Add an article: >>> import requests, json >>> headers = {'Content-type': 'application/json', 'Accept': 'application/json'} - >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/', 'title': 'Sortie de pyAggr3g470r 5.3', 'content':'La page principale de pyAggr3g470r a été améliorée...', 'date':'06/23/2014 11:42 AM', 'feed_id':'42'} - >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) - >>> print r.content - { - "message": "ok" - } - >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles?feed=42&limit=1", auth=("your-email", "your-password")) - >>> print json.loads(r.content)["result"][0]["title"] - Sortie de pyAggr3g470r 5.3 + >>> payload = {'link': 'http://blog.cedricbonhomme.org/2014/05/24/sortie-de-pyaggr3g470r-5-3/', + ... 'title': 'Sortie de pyAggr3g470r 5.3', + ... 'content':'La page principale de pyAggr3g470r a été améliorée...', + ... 'date':'2014/06/23T11:42:20 GMT', + ... 'feed_id':'42'} + >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/article", + ... headers=headers, auth=("your-nickname", "your-password"), + ... data=json.dumps(payload)) + >>> r.status_code + 201 # Created + >>> # creating several articles at once + >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/article", + ... headers=headers, auth=("your-nickname", "your-password"), + ... data=json.dumps([payload, payload])) + >>> r.status_code + 201 # Created + >>> r.json() + [123456, 234567] + >>> r = requests.get("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... auth=("your-nickname", "your-password") + ... data=json.dumps({'feed_id': 42, 'limit': 1})) + >>> r.json()[0]["title"] + "Sortie de pyAggr3g470r 5.3" Update an article: .. code-block:: python - >>> payload = {"like":True, "readed":False} - >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/65", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) - >>> print r.content - { - "message": "ok" - } - -Delete an article: + >>> import requests, json + >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/65", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password"), + ... data=json.dumps({"like":True, "readed": False})) + >>> r.status_code + 200 # OK + >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password"), + ... data=json.dumps([[1, {"like": True, "readed": False}], + ... [2, {"like": True, "readed": True}]])) + >>> r.json() + ['ok', 'ok'] + +Delete one or several article(s): .. code-block:: python - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password")) - >>> print r.status_code - 200 - >>> print r.content - { - "message": "ok" - } - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/articles/84574", auth=("your-email", "your-password")) - >>> print r.status_code - 200 - >>> print r.content - { - "message": "Article not found." - } + >>> import json, requests + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password")) + >>> r.status_code + 204 # deleted - No content + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/article/84574", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password")) + >>> r.status_code + 404 # not found + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password") + ... data=json.dumps([84574])) + >>> r.status_code + 500 # already deleted + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password") + ... data=json.dumps([84575, 84576])) + >>> r.status_code + 204 # deleted - No content + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/articles", + ... headers={'Content-Type': 'application/json'}, + ... auth=("your-nickname", "your-password") + ... data=json.dumps([84575, 84576, 84577])) + >>> r.status_code + 206 # partial - some deleted + >>> r.json() + ['404 - Not Found', '404 - Not Found', 'ok'] + Feeds ----- @@ -83,18 +118,30 @@ Add a feed: .. code-block:: python - >>> payload = {'link': 'http://blog.cedricbonhomme.org/feed'} - >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) + >>> import json, requests + >>> r = requests.post("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds", + ... auth=("your-nickname", "your-password"), + ... headers={'Content-Type': 'application/json'}, + ... data=json.dumps({'link': 'http://blog.cedricbonhomme.org/feed'})) + >>> r.status_code + 200 Update a feed: .. code-block:: python - >>> payload = {"title":"Feed new title", "description":"New description"} - >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/42", headers=headers, auth=("your-email", "your-password"), data=json.dumps(payload)) + >>> import json, requests + >>> r = requests.put("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/42", + ... auth=("your-nickname", "your-password"), + ... headers={'Content-Type': 'application/json'}, + ... data=json.dumps({"title":"Feed new title", "description":"New description"}) + >>> r.status_code + 201 Delete a feed: .. code-block:: python - >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v1.0/feeds/29", auth=("your-email", "your-password")) + >>> import requests + >>> r = requests.delete("https://pyaggr3g470r.herokuapp.com/api/v2.0/feeds/29", + ... auth=("your-nickname", "your-password")) diff --git a/manage.py b/manage.py deleted file mode 100644 index 056fef2c..00000000 --- a/manage.py +++ /dev/null @@ -1,15 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from flask.ext.script import Manager -from flask.ext.migrate import Migrate, MigrateCommand - -from pyaggr3g470r import app, db - -migrate = Migrate(app, db) -manager = Manager(app) - -manager.add_command('db', MigrateCommand) - -if __name__ == '__main__': - manager.run() diff --git a/manager.py b/manager.py new file mode 100755 index 00000000..89fd2bf1 --- /dev/null +++ b/manager.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +import os +from bootstrap import application, db, populate_g +from flask.ext.script import Manager +from flask.ext.migrate import Migrate, MigrateCommand + +from werkzeug import generate_password_hash + +from sqlalchemy.engine import reflection +from sqlalchemy.schema import ( + MetaData, + Table, + DropTable, + ForeignKeyConstraint, + DropConstraint) + + +Migrate(application, db) + +manager = Manager(application) +manager.add_command('db', MigrateCommand) + + +@manager.command +def db_empty(): + "will drop every datas stocked in db" + # From http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything + conn = db.engine.connect() + + # the transaction only applies if the DB supports + # transactional DDL, i.e. Postgresql, MS SQL Server + trans = conn.begin() + + inspector = reflection.Inspector.from_engine(db.engine) + + # gather all data first before dropping anything. + # some DBs lock after things have been dropped in + # a transaction. + metadata = MetaData() + + tbs = [] + all_fks = [] + + for table_name in inspector.get_table_names(): + fks = [] + for fk in inspector.get_foreign_keys(table_name): + if not fk['name']: + continue + fks.append(ForeignKeyConstraint((), (), name=fk['name'])) + t = Table(table_name, metadata, *fks) + tbs.append(t) + all_fks.extend(fks) + + for fkc in all_fks: + conn.execute(DropConstraint(fkc)) + + for table in tbs: + conn.execute(DropTable(table)) + + trans.commit() + + +@manager.command +def db_create(): + "Will create the database from conf parameters" + with application.app_context(): + populate_g() + from pyaggr3g470r.models import User, Role + db.create_all() + + role_admin = Role(name="admin") + role_user = Role(name="user") + + user1 = User(nickname="admin", + email=os.environ.get("ADMIN_EMAIL", + "root@pyAggr3g470r.localhost"), + pwdhash=generate_password_hash( + os.environ.get("ADMIN_PASSWORD", "password")), + activation_key="") + user1.roles.extend([role_admin, role_user]) + + db.session.add(user1) + db.session.commit() + + +@manager.command +def fetch(user, password, limit=10): + from pyaggr3g470r.lib.crawler import CrawlerScheduler + scheduler = CrawlerScheduler(user, password) + scheduler.run(limit=limit) + scheduler.wait() + + +if __name__ == '__main__': + manager.run() diff --git a/migrations/versions/1b750a389c22_remove_email_notification_column.py b/migrations/versions/1b750a389c22_remove_email_notification_column.py index 5ec748a8..71529855 100644 --- a/migrations/versions/1b750a389c22_remove_email_notification_column.py +++ b/migrations/versions/1b750a389c22_remove_email_notification_column.py @@ -10,13 +10,16 @@ Create Date: 2015-02-25 23:01:07.253429 revision = '1b750a389c22' down_revision = '48f561c0ce6' +import conf from alembic import op import sqlalchemy as sa def upgrade(): - op.drop_column('feed', 'email_notification') + if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: + op.drop_column('feed', 'email_notification') def downgrade(): - op.add_column('feed', sa.Column('email_notification', sa.Boolean(), default=False)) + op.add_column('feed', sa.Column('email_notification', sa.Boolean(), + default=False)) diff --git a/migrations/versions/48f561c0ce6_add_column_entry_id.py b/migrations/versions/48f561c0ce6_add_column_entry_id.py index 3f52a7a9..e5bc5735 100644 --- a/migrations/versions/48f561c0ce6_add_column_entry_id.py +++ b/migrations/versions/48f561c0ce6_add_column_entry_id.py @@ -1,7 +1,7 @@ """add column entry_id Revision ID: 48f561c0ce6 -Revises: +Revises: Create Date: 2015-02-18 21:17:19.346998 """ diff --git a/migrations/versions/4b5c161e1ced_.py b/migrations/versions/4b5c161e1ced_.py new file mode 100644 index 00000000..32cfe8c8 --- /dev/null +++ b/migrations/versions/4b5c161e1ced_.py @@ -0,0 +1,42 @@ +"""adding feed and user attributes for better feed retreiving + +Revision ID: 4b5c161e1ced +Revises: None +Create Date: 2015-01-17 01:04:10.187285 + +""" +from datetime import datetime + +# revision identifiers, used by Alembic. +revision = '4b5c161e1ced' +down_revision = '1b750a389c22' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + unix_start = datetime(1970, 1, 1) + # commands auto generated by Alembic - please adjust! ### + op.add_column('feed', sa.Column('error_count', sa.Integer(), nullable=True, + default=0, server_default="0")) + op.add_column('feed', sa.Column('last_error', sa.String(), nullable=True)) + op.add_column('feed', sa.Column('last_modified', sa.DateTime(), + nullable=True, default=unix_start, server_default=str(unix_start))) + op.add_column('feed', sa.Column('last_retreived', sa.DateTime(), + nullable=True, default=unix_start, server_default=str(unix_start))) + op.add_column('feed', sa.Column('etag', sa.String(), nullable=True)) + op.add_column('user', sa.Column('refresh_rate', sa.Integer(), + nullable=True)) + # end Alembic commands ### + + +def downgrade(): + # commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'refresh_rate') + op.drop_column('feed', 'last_modified') + op.drop_column('feed', 'last_error') + op.drop_column('feed', 'error_count') + op.drop_column('feed', 'last_retreived') + op.drop_column('feed', 'etag') + # end Alembic commands ### diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index f3f784f4..e69de29b 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -1,48 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -import os -from flask import Flask -from flask.ext.sqlalchemy import SQLAlchemy -from flask.ext.babel import Babel -from flask.ext.babel import format_datetime - -import conf - -# Create Flask application -app = Flask(__name__) -app.debug = conf.WEBSERVER_DEBUG - -# Create dummy secrey key so we can use sessions -app.config['SECRET_KEY'] = getattr(conf, 'WEBSERVER_SECRET', None) -if not app.config['SECRET_KEY']: - app.config['SECRET_KEY'] = os.urandom(12) -app.config['SQLALCHEMY_DATABASE_URI'] = conf.SQLALCHEMY_DATABASE_URI -db = SQLAlchemy(app) - -app.config['RECAPTCHA_USE_SSL'] = True -app.config['RECAPTCHA_PUBLIC_KEY'] = conf.RECAPTCHA_PUBLIC_KEY -app.config['RECAPTCHA_PRIVATE_KEY'] = conf.RECAPTCHA_PRIVATE_KEY - -if conf.ON_HEROKU: - from flask_sslify import SSLify - sslify = SSLify(app) - -ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) - -def allowed_file(filename): - """ - Check if the uploaded file is allowed. - """ - return '.' in filename and \ - filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS - -babel = Babel(app) - -app.jinja_env.filters['datetime'] = format_datetime - -# Views -from flask.ext.restful import Api -api = Api(app) - -from pyaggr3g470r import views, rest diff --git a/pyaggr3g470r/controllers/__init__.py b/pyaggr3g470r/controllers/__init__.py new file mode 100644 index 00000000..d8d1a104 --- /dev/null +++ b/pyaggr3g470r/controllers/__init__.py @@ -0,0 +1,6 @@ +from .feed import FeedController +from .article import ArticleController +from .user import UserController + + +__all__ = ['FeedController', 'ArticleController', 'UserController'] diff --git a/pyaggr3g470r/controllers/abstract.py b/pyaggr3g470r/controllers/abstract.py new file mode 100644 index 00000000..a99e67f3 --- /dev/null +++ b/pyaggr3g470r/controllers/abstract.py @@ -0,0 +1,69 @@ +import logging +from bootstrap import db +from sqlalchemy import update +from werkzeug.exceptions import Forbidden, NotFound + +logger = logging.getLogger(__name__) + + +class AbstractController(object): + _db_cls = None # reference to the database class + _user_id_key = 'user_id' + + def __init__(self, user_id): + self.user_id = user_id + + def _to_filters(self, **filters): + if self.user_id: + filters[self._user_id_key] = self.user_id + db_filters = set() + for key, value in filters.items(): + if key.endswith('__gt'): + db_filters.add(getattr(self._db_cls, key[:-4]) > value) + elif key.endswith('__lt'): + db_filters.add(getattr(self._db_cls, key[:-4]) < value) + elif key.endswith('__ge'): + db_filters.add(getattr(self._db_cls, key[:-4]) >= value) + elif key.endswith('__le'): + db_filters.add(getattr(self._db_cls, key[:-4]) <= value) + elif key.endswith('__ne'): + db_filters.add(getattr(self._db_cls, key[:-4]) != value) + elif key.endswith('__in'): + db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) + else: + db_filters.add(getattr(self._db_cls, key) == value) + return db_filters + + def _get(self, **filters): + return self._db_cls.query.filter(*self._to_filters(**filters)) + + def get(self, **filters): + obj = self._get(**filters).first() + if not obj: + raise NotFound({'message': 'No %r (%r)' + % (self._db_cls.__class__.__name__, filters)}) + if getattr(obj, self._user_id_key) != self.user_id: + raise Forbidden({'message': 'No authorized to access %r (%r)' + % (self._db_cls.__class__.__name__, filters)}) + return obj + + def create(self, **attrs): + attrs[self._user_id_key] = self.user_id + obj = self._db_cls(**attrs) + db.session.add(obj) + db.session.commit() + return obj + + def read(self, **filters): + return self._get(**filters) + + def update(self, filters, attrs): + result = self._get(**filters).update(attrs, synchronize_session=False) + db.session.commit() + return result + + def delete(self, obj_id): + obj = self.get(id=obj_id) + db.session.delete(obj) + db.session.commit() + return obj diff --git a/pyaggr3g470r/controllers/article.py b/pyaggr3g470r/controllers/article.py new file mode 100644 index 00000000..46ca0988 --- /dev/null +++ b/pyaggr3g470r/controllers/article.py @@ -0,0 +1,27 @@ +import conf +from .abstract import AbstractController +from pyaggr3g470r.models import Article + + +class ArticleController(AbstractController): + _db_cls = Article + + def get(self, **filters): + article = super(ArticleController, self).get(**filters) + if not article.readed: + self.update({'id': article.id}, {'readed': True}) + return article + + def delete(self, obj_id): + obj = super(ArticleController, self).delete(obj_id) + if not conf.ON_HEROKU: + import pyaggr3g470r.search as fastsearch + fastsearch.delete_article(self.user_id, obj.feed_id, obj_id) + return obj + + def challenge(self, ids): + """Will return each id that wasn't found in the database.""" + for id_ in ids: + if self.read(**id_).first(): + continue + yield id_ diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py new file mode 100644 index 00000000..a2455e2b --- /dev/null +++ b/pyaggr3g470r/controllers/feed.py @@ -0,0 +1,24 @@ +from datetime import datetime, timedelta +from .abstract import AbstractController +from pyaggr3g470r.models import Feed + +DEFAULT_MAX_ERROR = 3 +DEFAULT_LIMIT = 5 + + +class FeedController(AbstractController): + _db_cls = Feed + + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT): + from pyaggr3g470r.controllers import UserController + now = datetime.now() + user = UserController(self.user_id).get(id=self.user_id) + max_last = now - timedelta(minutes=user.refresh_rate or 60) + feeds = [feed for feed in self.read(user_id=self.user_id, + error_count__lt=max_error, enabled=True, + last_retreived__lt=max_last).limit(limit)] + + if feeds: + self.update({'id__in': [feed.id for feed in feeds]}, + {'last_retreived': now}) + return feeds diff --git a/pyaggr3g470r/controllers/user.py b/pyaggr3g470r/controllers/user.py new file mode 100644 index 00000000..c6c1d545 --- /dev/null +++ b/pyaggr3g470r/controllers/user.py @@ -0,0 +1,7 @@ +from .abstract import AbstractController +from pyaggr3g470r.models import User + + +class UserController(AbstractController): + _db_cls = User + _user_id_key = 'id' diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index 3e808793..9e8f9c0b 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -1,10 +1,12 @@ #! /usr/bin/env python -#-*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from threading import Thread from functools import wraps from flask import g, redirect, url_for, flash +from flask.ext.babel import gettext +from flask.ext.login import login_required from pyaggr3g470r.models import Feed @@ -20,6 +22,7 @@ def async(f): thr.start() return wrapper + def feed_access_required(func): """ This decorator enables to check if a user has access to a feed. @@ -37,3 +40,11 @@ def feed_access_required(func): else: return func(*args, **kwargs) return decorated + + +def pyagg_default_decorator(func): + @login_required + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper diff --git a/pyaggr3g470r/emails.py b/pyaggr3g470r/emails.py index 50f94761..9a129718 100644 --- a/pyaggr3g470r/emails.py +++ b/pyaggr3g470r/emails.py @@ -31,6 +31,7 @@ from pyaggr3g470r.decorators import async logger = logging.getLogger(__name__) + @async def send_async_email(mfrom, mto, msg): try: diff --git a/pyaggr3g470r/lib/__init__.py b/pyaggr3g470r/lib/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/pyaggr3g470r/lib/__init__.py diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py new file mode 100644 index 00000000..64ef8b6d --- /dev/null +++ b/pyaggr3g470r/lib/crawler.py @@ -0,0 +1,257 @@ +""" +Here's a sum up on how it works : + +CrawlerScheduler.run + will retreive a list of feeds to be refreshed and pass result to +CrawlerScheduler.callback + which will retreive each feed and treat result with +FeedCrawler.callback + which will interprete the result (status_code, etag) collect ids + and match them agaisnt pyagg which will cause +PyAggUpdater.callback + to create the missing entries +""" + +import time +import conf +import json +import logging +import requests +import feedparser +import dateutil.parser +from functools import wraps +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor +from requests_futures.sessions import FuturesSession +from pyaggr3g470r.lib.utils import default_handler + +logger = logging.getLogger(__name__) +API_ROOT = "api/v2.0/" + + +def extract_id(entry, keys=[('link', 'link'), + ('published', 'retrieved_date'), + ('updated', 'retrieved_date')], force_id=False): + """For a given entry will return a dict that allows to identify it. The + dict will be constructed on the uid of the entry. if that identifier is + absent, the dict will be constructed upon the values of "keys". + """ + entry_id = entry.get('entry_id') or entry.get('id') + if entry_id: + return {'entry_id': entry_id} + if not entry_id and force_id: + entry_id = hash("".join(entry[entry_key] for _, entry_key in keys + if entry_key in entry)) + else: + ids = {} + for entry_key, pyagg_key in keys: + if entry_key in entry and pyagg_key not in ids: + ids[pyagg_key] = entry[entry_key] + if 'date' in pyagg_key: + ids[pyagg_key] = dateutil.parser.parse(ids[pyagg_key])\ + .isoformat() + return ids + + +class AbstractCrawler: + __session__ = None + __counter__ = 0 + + def __init__(self, auth): + self.auth = auth + self.session = self.get_session() + self.url = conf.PLATFORM_URL + + @classmethod + def get_session(cls): + """methods that allows us to treat session as a singleton""" + if cls.__session__ is None: + cls.__session__ = FuturesSession( + executor=ThreadPoolExecutor(max_workers=conf.NB_WORKER)) + cls.__session__.verify = False + return cls.__session__ + + @classmethod + def count_on_me(cls, func): + """A basic decorator which will count +1 at the begining of a call + and -1 at the end. It kinda allows us to wait for the __counter__ value + to be 0, meaning nothing is done anymore.""" + @wraps(func) + def wrapper(*args, **kwargs): + cls.__counter__ += 1 + result = func(*args, **kwargs) + cls.__counter__ -= 1 + return result + return wrapper + + def query_pyagg(self, method, urn, data=None): + """A wrapper for internal call, method should be ones you can find + on requests (header, post, get, options, ...), urn the distant + resources you want to access on pyagg, and data, the data you wanna + transmit.""" + if data is None: + data = {} + method = getattr(self.session, method) + return method("%s%s%s" % (self.url, API_ROOT, urn), + auth=self.auth, data=json.dumps(data, + default=default_handler), + headers={'Content-Type': 'application/json'}) + + @classmethod + def wait(cls): + "See count_on_me, that method will just wait for the counter to be 0" + time.sleep(1) + while cls.__counter__: + time.sleep(1) + + +class PyAggUpdater(AbstractCrawler): + + def __init__(self, feed, entries, headers, auth): + self.feed = feed + self.entries = entries + self.headers = headers + super(PyAggUpdater, self).__init__(auth) + + def to_article(self, entry): + "Safe method to transorm a feedparser entry into an article" + date = datetime.now() + + for date_key in ('published', 'updated'): + if entry.get(date_key): + try: + date = dateutil.parser.parse(entry[date_key]) + except Exception: + pass + else: + break + content = '' + if entry.get('content'): + content = entry['content'][0]['value'] + elif entry.get('summary'): + content = entry['summary'] + + return {'feed_id': self.feed['id'], + 'entry_id': extract_id(entry).get('entry_id', None), + 'link': entry.get('link', self.feed['site_link']), + 'title': entry.get('title', 'No title'), + 'readed': False, 'like': False, + 'content': content, + 'retrieved_date': date.isoformat(), + 'date': date.isoformat()} + + @AbstractCrawler.count_on_me + def callback(self, response): + """Will process the result from the challenge, creating missing article + and updating the feed""" + results = response.result().json() + logger.debug('%r %r - %d entries were not matched and will be created', + self.feed['id'], self.feed['title'], len(results)) + for id_to_create in results: + entry = self.to_article( + self.entries[tuple(sorted(id_to_create.items()))]) + logger.info('creating %r - %r', entry['title'], id_to_create) + self.query_pyagg('post', 'article', entry) + + now = datetime.now() + logger.debug('%r %r - updating feed etag %r last_mod %r', + self.feed['id'], self.feed['title'], + self.headers.get('etag'), now) + + self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': 0, + 'etag': self.headers.get('etag', ''), + 'last_error': '', + 'last_modified': self.headers.get('last-modified', '')}) + + +class FeedCrawler(AbstractCrawler): + + def __init__(self, feed, auth): + self.feed = feed + super(FeedCrawler, self).__init__(auth) + + def clean_feed(self): + """Will reset the errors counters on a feed that have known errors""" + if self.feed.get('error_count') or self.feed.get('last_error'): + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': 0, 'last_error': ''}) + + @AbstractCrawler.count_on_me + def callback(self, response): + """will fetch the feed and interprete results (304, etag) or will + challenge pyagg to compare gotten entries with existing ones""" + try: + response = response.result() + response.raise_for_status() + except Exception as error: + error_count = self.feed['error_count'] + 1 + logger.warn('%r %r - an error occured while fetching feed; bumping' + ' error count to %r', self.feed['id'], + self.feed['title'], error_count) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': error_count, + 'last_error': str(error)}) + return + + if response.status_code == 304: + logger.info("%r %r - feed responded with 304", + self.feed['id'], self.feed['title']) + self.clean_feed() + return + if self.feed['etag'] and response.headers.get('etag') \ + and response.headers.get('etag') == self.feed['etag']: + logger.info("%r %r - feed responded with same etag (%d)", + self.feed['id'], self.feed['title'], + response.status_code) + self.clean_feed() + return + ids, entries = [], {} + parsed_response = feedparser.parse(response.text) + for entry in parsed_response['entries']: + entries[tuple(sorted(extract_id(entry).items()))] = entry + ids.append(extract_id(entry)) + logger.debug('%r %r - found %d entries %r', + self.feed['id'], self.feed['title'], len(ids), ids) + future = self.query_pyagg('get', 'articles/challenge', {'ids': ids}) + updater = PyAggUpdater(self.feed, entries, response.headers, self.auth) + future.add_done_callback(updater.callback) + + +class CrawlerScheduler(AbstractCrawler): + + def __init__(self, username, password): + self.auth = (username, password) + super(CrawlerScheduler, self).__init__(self.auth) + + def prepare_headers(self, feed): + """For a known feed, will construct some header dictionnary""" + headers = {} + if feed.get('etag', None): + headers['If-None-Match'] = feed['etag'] + if feed.get('last_modified'): + headers['If-Modified-Since'] = feed['last_modified'] + logger.debug('%r %r - calculated headers %r', + feed['id'], feed['title'], headers) + return headers + + @AbstractCrawler.count_on_me + def callback(self, response): + """processes feeds that need to be fetched""" + response = response.result() + response.raise_for_status() + feeds = response.json() + logger.debug('%d to fetch %r', len(feeds), feeds) + for feed in feeds: + logger.info('%r %r - fetching resources', + feed['id'], feed['title']) + future = self.session.get(feed['link'], + headers=self.prepare_headers(feed)) + future.add_done_callback(FeedCrawler(feed, self.auth).callback) + + @AbstractCrawler.count_on_me + def run(self, **kwargs): + """entry point, will retreive feeds to be fetch + and launch the whole thing""" + logger.debug('retreving fetchable feed') + future = self.query_pyagg('get', 'feeds/fetchable', kwargs) + future.add_done_callback(self.callback) diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py new file mode 100644 index 00000000..a4f4b3ec --- /dev/null +++ b/pyaggr3g470r/lib/utils.py @@ -0,0 +1,14 @@ +import types + +def default_handler(obj): + """JSON handler for default query formatting""" + if hasattr(obj, 'isoformat'): + return obj.isoformat() + if hasattr(obj, 'dump'): + return obj.dump() + if isinstance(obj, (set, frozenset, types.GeneratorType)): + return list(obj) + if isinstance(obj, BaseException): + return str(obj) + raise TypeError("Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj)) diff --git a/pyaggr3g470r/models.py b/pyaggr3g470r/models.py deleted file mode 100644 index 8bc9e76b..00000000 --- a/pyaggr3g470r/models.py +++ /dev/null @@ -1,152 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.4 $" -__date__ = "$Date: 2013/11/05 $" -__revision__ = "$Date: 2014/04/12 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "GPLv3" - -import re -import json -import random, hashlib -from datetime import datetime -from sqlalchemy import asc, desc -from werkzeug import generate_password_hash, check_password_hash -from flask.ext.login import UserMixin - -from pyaggr3g470r import db - -class User(db.Model, UserMixin): - """ - Represent a user. - """ - id = db.Column(db.Integer, primary_key = True) - nickname = db.Column(db.String(), unique = True) - email = db.Column(db.String(254), index = True, unique = True) - pwdhash = db.Column(db.String()) - roles = db.relationship('Role', backref = 'user', lazy = 'dynamic') - activation_key = db.Column(db.String(128), default = - hashlib.sha512( - str(random.getrandbits(256)).encode("utf-8") - ).hexdigest()[:86]) - date_created = db.Column(db.DateTime(), default=datetime.now) - last_seen = db.Column(db.DateTime(), default=datetime.now) - feeds = db.relationship('Feed', backref = 'subscriber', lazy = 'dynamic', cascade='all,delete-orphan') - - @staticmethod - def make_valid_nickname(nickname): - return re.sub('[^a-zA-Z0-9_\.]', '', nickname) - - def get_id(self): - """ - Return the id (email) of the user. - """ - return self.email - - def set_password(self, password): - """ - Hash the password of the user. - """ - self.pwdhash = generate_password_hash(password) - - def check_password(self, password): - """ - Check the password of the user. - """ - return check_password_hash(self.pwdhash, password) - - def is_admin(self): - """ - Return True if the user has administrator rights. - """ - return len([role for role in self.roles if role.name == "admin"]) != 0 - - def __eq__(self, other): - return self.id == other.id - - def __repr__(self): - return '<User %r>' % (self.nickname) - -class Role(db.Model): - """ - Represent a role. - """ - id = db.Column(db.Integer, primary_key = True) - name = db.Column(db.String(), unique = True) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - -class Feed(db.Model): - """ - Represent a feed. - """ - id = db.Column(db.Integer, primary_key = True) - title = db.Column(db.String(), default="No title") - description = db.Column(db.String(), default="FR") - link = db.Column(db.String()) - site_link = db.Column(db.String(), default="") - enabled = db.Column(db.Boolean(), default=True) - created_date = db.Column(db.DateTime(), default=datetime.now) - articles = db.relationship('Article', backref = 'source', lazy = 'dynamic', cascade='all,delete-orphan', - order_by=desc("Article.date")) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - - def __repr__(self): - return '<Feed %r>' % (self.title) - -class Article(db.Model): - """ - Represent an article from a feed. - """ - id = db.Column(db.Integer, primary_key = True) - entry_id = db.Column(db.String()) - link = db.Column(db.String()) - title = db.Column(db.String()) - content = db.Column(db.String()) - readed = db.Column(db.Boolean(), default=False) - like = db.Column(db.Boolean(), default=False) - date = db.Column(db.DateTime(), default=datetime.now) - retrieved_date = db.Column(db.DateTime(), default=datetime.now) - - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) - - def previous_article(self): - """ - Returns the previous article (older). - """ - return Article.query.filter(Article.date < self.date, Article.feed_id == self.feed_id).order_by(desc("Article.date")).first() - - def next_article(self): - """ - Returns the next article (newer). - """ - return Article.query.filter(Article.date > self.date, Article.feed_id == self.feed_id).order_by(asc("Article.date")).first() - - def __repr__(self): - return json.dumps({ - "title": self.title, - "link": self.link, - "content": self.content - }) diff --git a/pyaggr3g470r/models/__init__.py b/pyaggr3g470r/models/__init__.py new file mode 100644 index 00000000..9584d1f2 --- /dev/null +++ b/pyaggr3g470r/models/__init__.py @@ -0,0 +1,35 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from .feed import Feed +from .role import Role +from .user import User +from .article import Article + + +__all__ = ['Feed', 'Role', 'User', 'Article'] diff --git a/pyaggr3g470r/models/article.py b/pyaggr3g470r/models/article.py new file mode 100644 index 00000000..0466bc35 --- /dev/null +++ b/pyaggr3g470r/models/article.py @@ -0,0 +1,87 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +import json +from datetime import datetime +from flask import g +from sqlalchemy import asc, desc + +db = g.db + + +class Article(db.Model): + """ + Represent an article from a feed. + """ + id = db.Column(db.Integer, primary_key=True) + entry_id = db.Column(db.String()) + link = db.Column(db.String()) + title = db.Column(db.String()) + content = db.Column(db.String()) + readed = db.Column(db.Boolean(), default=False) + like = db.Column(db.Boolean(), default=False) + date = db.Column(db.DateTime(), default=datetime.now) + retrieved_date = db.Column(db.DateTime(), default=datetime.now) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) + + def previous_article(self): + """ + Returns the previous article (older). + """ + return Article.query.filter(Article.date < self.date, + Article.feed_id == self.feed_id)\ + .order_by(desc("Article.date")).first() + + def next_article(self): + """ + Returns the next article (newer). + """ + return Article.query.filter(Article.date > self.date, + Article.feed_id == self.feed_id)\ + .order_by(asc("Article.date")).first() + + def __repr__(self): + return json.dumps({ + "title": self.title, + "link": self.link, + "content": self.content + }) + + def dump(self): + return {"id": self.id, + "title": self.title, + "link": self.link, + "content": self.content, + "readed": self.readed, + "like": self.like, + "date": self.date, + "retrieved_date": self.retrieved_date, + "feed_id": getattr(self.source, 'id', None), + "feed_name": getattr(self.source, 'title', None)} diff --git a/pyaggr3g470r/models/feed.py b/pyaggr3g470r/models/feed.py new file mode 100644 index 00000000..28cb2b92 --- /dev/null +++ b/pyaggr3g470r/models/feed.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from datetime import datetime +from flask import g +from sqlalchemy import desc + +db = g.db + + +class Feed(db.Model): + """ + Represent a station. + """ + id = db.Column(db.Integer, primary_key=True) + title = db.Column(db.String(), default="No title") + description = db.Column(db.String(), default="FR") + link = db.Column(db.String()) + site_link = db.Column(db.String(), default="") + enabled = db.Column(db.Boolean(), default=True) + created_date = db.Column(db.DateTime(), default=datetime.now) + + # cache handling + etag = db.Column(db.String(), default="") + last_modified = db.Column(db.String(), default="") + last_retreived = db.Column(db.DateTime(), default=datetime(1970, 1, 1)) + + # error logging + last_error = db.Column(db.String(), default="") + error_count = db.Column(db.Integer(), default=0) + + # relationship + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + articles = db.relationship('Article', backref='source', lazy='dynamic', + cascade='all,delete-orphan', + order_by=desc("Article.date")) + + def __repr__(self): + return '<Feed %r>' % (self.title) + + def dump(self): + return {"id": self.id, + "title": self.title, + "description": self.description, + "link": self.link, + "site_link": self.site_link, + "etag": self.etag, + "error_count": self.error_count, + "last_modified": self.last_modified, + "last_retreived": self.last_retreived} diff --git a/pyaggr3g470r/models/role.py b/pyaggr3g470r/models/role.py new file mode 100644 index 00000000..71497caf --- /dev/null +++ b/pyaggr3g470r/models/role.py @@ -0,0 +1,41 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +from flask import g + +db = g.db + + +class Role(db.Model): + """ + Represent a role. + """ + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(), unique=True) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) diff --git a/pyaggr3g470r/models/user.py b/pyaggr3g470r/models/user.py new file mode 100644 index 00000000..f2a268db --- /dev/null +++ b/pyaggr3g470r/models/user.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - https://www.cedricbonhomme.org +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.4 $" +__date__ = "$Date: 2013/11/05 $" +__revision__ = "$Date: 2014/04/12 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "GPLv3" + +import re +import random +import hashlib +from datetime import datetime +from flask import g +from werkzeug import generate_password_hash, check_password_hash +from flask.ext.login import UserMixin + +db = g.db + + +class User(db.Model, UserMixin): + """ + Represent a user. + """ + id = db.Column(db.Integer, primary_key=True) + nickname = db.Column(db.String(), unique=True) + email = db.Column(db.String(254), index=True, unique=True) + pwdhash = db.Column(db.String()) + roles = db.relationship('Role', backref='user', lazy='dynamic') + activation_key = db.Column(db.String(128), default=hashlib.sha512( + str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86]) + date_created = db.Column(db.DateTime(), default=datetime.now) + last_seen = db.Column(db.DateTime(), default=datetime.now) + feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic', + cascade='all,delete-orphan') + refresh_rate = db.Column(db.Integer, default=60) # in minutes + + @staticmethod + def make_valid_nickname(nickname): + return re.sub('[^a-zA-Z0-9_\.]', '', nickname) + + def get_id(self): + """ + Return the id (email) of the user. + """ + return self.email + + def set_password(self, password): + """ + Hash the password of the user. + """ + self.pwdhash = generate_password_hash(password) + + def check_password(self, password): + """ + Check the password of the user. + """ + return check_password_hash(self.pwdhash, password) + + def is_admin(self): + """ + Return True if the user has administrator rights. + """ + return len([role for role in self.roles if role.name == "admin"]) != 0 + + def __eq__(self, other): + return self.id == other.id + + def __repr__(self): + return '<User %r>' % (self.nickname) diff --git a/pyaggr3g470r/notifications.py b/pyaggr3g470r/notifications.py index 1acf782e..cf8fb723 100644 --- a/pyaggr3g470r/notifications.py +++ b/pyaggr3g470r/notifications.py @@ -19,9 +19,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from pyaggr3g470r import conf +import conf from pyaggr3g470r import emails + def information_message(subject, plaintext): """ Send an information message to the users of the platform. diff --git a/pyaggr3g470r/rest.py b/pyaggr3g470r/rest.py deleted file mode 100644 index 1f354167..00000000 --- a/pyaggr3g470r/rest.py +++ /dev/null @@ -1,351 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# pyAggr3g470r - A Web based news aggregator. -# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ -# -# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -__author__ = "Cedric Bonhomme" -__version__ = "$Revision: 0.2 $" -__date__ = "$Date: 2014/06/18 $" -__revision__ = "$Date: 2014/07/05 $" -__copyright__ = "Copyright (c) Cedric Bonhomme" -__license__ = "AGPLv3" - -import re -import dateutil.parser -from functools import wraps -from flask import g, Response, request, session, jsonify -from flask.ext.restful import Resource, reqparse - -import conf -if not conf.ON_HEROKU: - import pyaggr3g470r.search as fastsearch -from pyaggr3g470r import api, db -from pyaggr3g470r.models import User, Article, Feed - -def authenticate(func): - """ - Decorator for the authentication to the web services. - """ - @wraps(func) - def wrapper(*args, **kwargs): - if not getattr(func, 'authenticated', True): - return func(*args, **kwargs) - - # authentication based on the session (already logged on the site) - if 'email' in session or g.user.is_authenticated(): - return func(*args, **kwargs) - - # authentication via HTTP only - auth = request.authorization - try: - email = auth.username - user = User.query.filter(User.email == email).first() - if user and user.check_password(auth.password) and user.activation_key == "": - g.user = user - return func(*args, **kwargs) - except AttributeError: - pass - - return Response('<Authentication required>', 401, - {'WWWAuthenticate':'Basic realm="Login Required"'}) - return wrapper - -class ArticleListAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, location = 'json') - self.reqparse.add_argument('content', type = unicode, location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('date', type = str, location = 'json') - self.reqparse.add_argument('feed_id', type = int, location = 'json') - super(ArticleListAPI, self).__init__() - - def get(self): - """ - Returns a list of articles. - """ - feeds = {feed.id: feed.title for feed in g.user.feeds if feed.enabled} - articles = Article.query.filter(Article.feed_id.in_(feeds.keys()), - Article.user_id == g.user.id) - filter_ = request.args.get('filter_', 'unread') - feed_id = int(request.args.get('feed', 0)) - limit = request.args.get('limit', 1000) - if filter_ != 'all': - articles = articles.filter(Article.readed == (filter_ == 'read')) - if feed_id: - articles = articles.filter(Article.feed_id == feed_id) - - articles = articles.order_by(Article.date.desc()) - if limit != 'all': - limit = int(limit) - articles = articles.limit(limit) - - return jsonify(result= [{ - "id": article.id, - "title": article.title, - "link": article.link, - "content": article.content, - "readed": article.readed, - "like": article.like, - "date": article.date, - "retrieved_date": article.retrieved_date, - "feed_id": article.source.id, - "feed_name": article.source.title - } - for article in articles] - ) - - def post(self): - """ - POST method - Create a new article. - """ - args = self.reqparse.parse_args() - article_dict = {} - for k, v in args.iteritems(): - if v != None: - article_dict[k] = v - else: - return {"message":"Missing argument: %s." % (k,)} - article_date = None - try: - article_date = dateutil.parser.parse(article_dict["date"], dayfirst=True) - except: - try: # trying to clean date field from letters - article_date = dateutil.parser.parse(re.sub('[A-z]', '', article_dict["date"], dayfirst=True)) - except: - return jsonify({"message":"Bad format for the date."}) - article = Article(link=article_dict["link"], title=article_dict["title"], - content=article_dict["content"], readed=False, like=False, - date=article_date, user_id=g.user.id, - feed_id=article_dict["feed_id"]) - feed = Feed.query.filter(Feed.id == article_dict["feed_id"], Feed.user_id == g.user.id).first() - feed.articles.append(article) - try: - db.session.commit() - return jsonify({"message":"ok"}) - except: - return jsonify({"message":"Impossible to create the article."}) - -class ArticleAPI(Resource): - """ - Defines a RESTful API for Article elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('like', type = bool, location = 'json') - self.reqparse.add_argument('readed', type = bool, location = 'json') - super(ArticleAPI, self).__init__() - - def get(self, id=None): - """ - Returns an article. - """ - result = [] - if id is not None: - article = Article.query.filter(Article.user_id == g.user.id, Article.id == id).first() - if article is not None: - if not article.readed: - article.readed = True - db.session.commit() - result.append(article) - - return jsonify(result= [{ - "id": article.id, - "title": article.title, - "link": article.link, - "content": article.content, - "readed": article.readed, - "like": article.like, - "date": article.date, - "retrieved_date": article.retrieved_date, - "feed_id": article.source.id, - "feed_name": article.source.title - } - for article in result] - ) - - def put(self, id): - """ - Update an article. - It is only possible to update the status ('like' and 'readed') of an article. - """ - args = self.reqparse.parse_args() - article = Article.query.filter(Article.id == id).first() - if article is not None and article.source.subscriber.id == g.user.id: - if None is not args.get('like', None): - article.like = args['like'] - if None is not args.get('readed', None): - article.readed = args['readed'] - db.session.commit() - - try: - fastsearch.delete_article(g.user.id, article.feed_id, article.id) - except: - pass - - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Article not found.'}) - - def delete(self, id): - """ - Delete an article. - """ - article = Article.query.filter(Article.id == id).first() - if article is not None and article.source.subscriber.id == g.user.id: - db.session.delete(article) - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Article not found.'}) - -api.add_resource(ArticleListAPI, '/api/v1.0/articles', endpoint = 'articles.json') -api.add_resource(ArticleAPI, '/api/v1.0/articles/<int:id>', endpoint = 'article.json') - -class FeedListAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('description', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('site_link', type = unicode, default = "", location = 'json') - self.reqparse.add_argument('enabled', type = bool, default = True ,location = 'json') - super(FeedListAPI, self).__init__() - - def get(self): - """ - Returns a list of feeds. - """ - return jsonify(result= [{ - "id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "enabled": feed.enabled, - "created_date": feed.created_date - } - for feed in g.user.feeds] - ) - - def post(self): - """ - POST method - Create a new feed. - """ - args = self.reqparse.parse_args() - feed_dict = {} - for k, v in args.iteritems(): - if v != None: - feed_dict[k] = v - else: - return jsonify({'message': 'missing argument: %s' % (k,)}) - new_feed = Feed(title=feed_dict["title"], description=feed_dict["description"], - link=feed_dict["link"], site_link=feed_dict["site_link"], - enabled=feed_dict["enabled"]) - g.user.feeds.append(new_feed) - try: - db.session.commit() - return jsonify({"message":"ok"}) - except: - return jsonify({'message': 'Impossible to create the feed.'}) - -class FeedAPI(Resource): - """ - Defines a RESTful API for Feed elements. - """ - method_decorators = [authenticate] - - def __init__(self): - self.reqparse = reqparse.RequestParser() - self.reqparse.add_argument('title', type = unicode, location = 'json') - self.reqparse.add_argument('description', type = unicode, location = 'json') - self.reqparse.add_argument('link', type = unicode, location = 'json') - self.reqparse.add_argument('site_link', type = unicode, location = 'json') - self.reqparse.add_argument('enabled', type = bool ,location = 'json') - super(FeedAPI, self).__init__() - - def get(self, id=None): - """ - Returns a feed. - """ - result = [] - if id is not None: - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - result.append(feed) - return jsonify(result= [{ - "id": feed.id, - "title": feed.title, - "description": feed.description, - "link": feed.link, - "site_link": feed.site_link, - "nb_articles": feed.articles.count() - } - for feed in result] - ) - return jsonify({'message': 'Feed not found'}) - - def put(self, id): - """ - Update a feed. - """ - args = self.reqparse.parse_args() - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - if None is not args.get('title', None): - feed.title = args['title'] - if None is not args.get('description', None): - feed.description = args['description'] - if None is not args.get('link', None): - feed.link = args['link'] - if None is not args.get('site_link', None): - feed.site_link = args['site_link'] - if None is not args.get('enabled', None): - feed.enabled = args['enabled'] - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Feed not found.'}) - - def delete(self, id): - """ - Delete a feed. - """ - feed = Feed.query.filter(Feed.id == id, Feed.user_id == g.user.id).first() - if feed is not None: - db.session.delete(feed) - db.session.commit() - return jsonify({"message":"ok"}) - else: - return jsonify({'message': 'Feed not found.'}) - -api.add_resource(FeedListAPI, '/api/v1.0/feeds', endpoint = 'feeds.json') -api.add_resource(FeedAPI, '/api/v1.0/feeds/<int:id>', endpoint = 'feed.json')
\ No newline at end of file diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py index 89fa0860..a7f780df 100644 --- a/pyaggr3g470r/search.py +++ b/pyaggr3g470r/search.py @@ -102,7 +102,7 @@ def delete_article(user_id, feed_id, article_id): try: ix = open_dir(indexdir) except (EmptyIndexError, OSError): - raise EmptyIndexError + return writer = ix.writer() document = And([Term("user_id", user_id), Term("feed_id", feed_id), Term("article_id", article_id)]) diff --git a/pyaggr3g470r/static/js/articles.js b/pyaggr3g470r/static/js/articles.js index 51273f3d..312a5cb6 100644 --- a/pyaggr3g470r/static/js/articles.js +++ b/pyaggr3g470r/static/js/articles.js @@ -18,6 +18,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +API_ROOT = 'api/v2.0/' + if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } +function ($) { @@ -78,7 +80,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // Encode your data as JSON. data: data, // This is the type of data you're expecting back from the server. - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, @@ -114,7 +116,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // Encode your data as JSON. data: data, // This is the type of data you're expecting back from the server. - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, @@ -132,7 +134,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Requires jQuery') } // sends the updates to the server $.ajax({ type: 'DELETE', - url: "/api/v1.0/articles/"+article_id, + url: API_ROOT + "article/" + article_id, success: function (result) { //console.log(result); }, diff --git a/pyaggr3g470r/templates/feed.html b/pyaggr3g470r/templates/feed.html index 21db7ebe..9910ccf7 100644 --- a/pyaggr3g470r/templates/feed.html +++ b/pyaggr3g470r/templates/feed.html @@ -14,6 +14,16 @@ ({{ ((feed.articles.all()|count * 100 ) / nb_articles) | round(2, 'floor') }}% {{ _('of the database') }}) {% endif %} .<br /> + {% if feed.error_count > 2 %} + <b>{{ _("That feed has encountered too much consecutive errors and won't be retreived anymore") }}</b> + {% elif feed.error_count > 0 %} + {{ _("That feed has encountered some errors but that counter will be reinitialized at the next successful retreiving") }} + {% endif %} + .<br /> + {% if feed.last_error %} + {{ _("Here's the last error encountered while retreiving this feed:") }} <pre>{{ feed.last_error }}</pre> + {% endif %} + {{ _('Address of the feed') }}: <a href="{{ feed.link }}">{{ feed.link }}</a><br /> {% if feed.site_link != "" %} {{ _('Address of the site') }}: <a href="{{ feed.site_link }}">{{ feed.site_link }}</a> diff --git a/pyaggr3g470r/templates/home.html b/pyaggr3g470r/templates/home.html index a00f962e..8170a99d 100644 --- a/pyaggr3g470r/templates/home.html +++ b/pyaggr3g470r/templates/home.html @@ -21,6 +21,9 @@ {% for fid, nbunread in unread|dictsort(by='value')|reverse %} <li class="feed-menu"><a href="{{ gen_url(feed=fid) }}"> {% if feed_id == fid %}<b>{% endif %} + {% if in_error.get(fid, 0) > 0 %} + <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span> + {% endif %} <span id="unread-{{ fid }}" class="badge pull-right">{{ nbunread }}</span> {{ feeds[fid]|safe }} {% if feed_id == fid %}</b>{% endif %} @@ -36,6 +39,9 @@ {% endfor %} {% for fid, ftitle in feeds|dictsort(case_sensitive=False, by='value') if not fid in unread %} <li class="feed-menu"><a href="{{ gen_url(feed=fid) }}"> + {% if in_error.get(fid, 0) > 0 %} + <span style="background-color: {{ "red" if in_error[fid] > 2 else "orange" }} ;" class="badge pull-right" title="some errors occured while trying to retreive that feed">{{ in_error[fid] }}</span> + {% endif %} {% if feed_id == fid %}<b>{% endif %} {{ ftitle|safe }} {% if feed_id == fid %}</b>{% endif %} diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html index 4dc62350..6b929bf3 100644 --- a/pyaggr3g470r/templates/layout.html +++ b/pyaggr3g470r/templates/layout.html @@ -7,9 +7,9 @@ <meta name="description" content="pyAggr3g470r is a web-based news aggregator." /> <meta name="author" content="" /> <title>{% if head_title %}{{ head_title }} - {% endif %}pyAggr3g470r</title> - <link rel="shortcut icon" href="{{ url_for('.static', filename='img/favicon.png') }}" /> + <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.png') }}" /> <!-- Bootstrap core CSS --> - <link href="{{ url_for('.static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" /> + <link href="{{ url_for('static', filename = 'css/bootstrap.css') }}" rel="stylesheet" media="screen" /> <!-- Add custom CSS here --> <style> body { @@ -155,9 +155,9 @@ <!-- Bootstrap core JavaScript --> <!-- Placed at the end of the document so the pages load faster --> - <script src="{{ url_for('.static', filename = 'js/jquery.js') }}"></script> - <script src="{{ url_for('.static', filename = 'js/bootstrap.js') }}"></script> - <script src="{{ url_for('.static', filename = 'js/articles.js') }}"></script> + <script src="{{ url_for('static', filename = 'js/jquery.js') }}"></script> + <script src="{{ url_for('static', filename = 'js/bootstrap.js') }}"></script> + <script src="{{ url_for('static', filename = 'js/articles.js') }}"></script> <script type="text/javascript" class="source"> if (window.location.href.indexOf("filter_=all") > -1){ $("#tab-all").attr('class', "active"); diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py index 972909af..1fc84ff4 100755 --- a/pyaggr3g470r/utils.py +++ b/pyaggr3g470r/utils.py @@ -51,9 +51,10 @@ from collections import Counter from contextlib import contextmanager import conf -from pyaggr3g470r import db +from flask import g from pyaggr3g470r.models import User, Feed, Article + # regular expression to check URL url_finders = [ re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \ @@ -135,7 +136,7 @@ def import_opml(email, opml_content): return nb nb = read(subscriptions) - db.session.commit() + g.db.session.commit() return nb def import_json(email, json_content): @@ -158,7 +159,7 @@ def import_json(email, json_content): enabled=feed["enabled"]) user.feeds.append(new_feed) nb_feeds += 1 - db.session.commit() + g.db.session.commit() # Create articles for feed in json_account["result"]: @@ -178,7 +179,7 @@ def import_json(email, json_content): user_feed.articles.append(new_article) nb_articles += 1 - db.session.commit() + g.db.session.commit() return nb_feeds, nb_articles diff --git a/pyaggr3g470r/views/__init__.py b/pyaggr3g470r/views/__init__.py new file mode 100644 index 00000000..029dcb7d --- /dev/null +++ b/pyaggr3g470r/views/__init__.py @@ -0,0 +1,5 @@ +from .views import * +from .api import * + +from .article import article_bp, articles_bp +from .feed import feed_bp, feeds_bp diff --git a/pyaggr3g470r/views/api/__init__.py b/pyaggr3g470r/views/api/__init__.py new file mode 100644 index 00000000..e11cdd95 --- /dev/null +++ b/pyaggr3g470r/views/api/__init__.py @@ -0,0 +1,31 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# pyAggr3g470r - A Web based news aggregator. +# Copyright (C) 2010-2015 Cédric Bonhomme - http://cedricbonhomme.org/ +# +# For more information : https://bitbucket.org/cedricbonhomme/pyaggr3g470r/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +__author__ = "Cedric Bonhomme" +__version__ = "$Revision: 0.2 $" +__date__ = "$Date: 2014/06/18 $" +__revision__ = "$Date: 2014/07/05 $" +__copyright__ = "Copyright (c) Cedric Bonhomme" +__license__ = "AGPLv3" + +from pyaggr3g470r.views.api import article, feed + +__all__ = ['article', 'feed'] diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py new file mode 100644 index 00000000..17881412 --- /dev/null +++ b/pyaggr3g470r/views/api/article.py @@ -0,0 +1,58 @@ +from flask import g +import dateutil.parser + +from pyaggr3g470r.controllers import ArticleController +from pyaggr3g470r.views.api.common import PyAggAbstractResource,\ + PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti + + +ARTICLE_ATTRS = {'feed_id': {'type': str}, + 'entry_id': {'type': str}, + 'link': {'type': str}, + 'title': {'type': str}, + 'readed': {'type': bool}, 'like': {'type': bool}, + 'content': {'type': str}, + 'date': {'type': str}, 'retrieved_date': {'type': str}} + + +class ArticleNewAPI(PyAggResourceNew): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] + + +class ArticleAPI(PyAggResourceExisting): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] + + +class ArticlesAPI(PyAggResourceMulti): + controller_cls = ArticleController + attrs = ARTICLE_ATTRS + to_date = ['date', 'retrieved_date'] + + +class ArticlesChallenge(PyAggAbstractResource): + controller_cls = ArticleController + attrs = {'ids': {'type': list, 'default': []}} + to_date = ['date', 'retrieved_date'] + + def get(self): + parsed_args = self.reqparse_args() + for id_dict in parsed_args['ids']: + for key in self.to_date: + if key in id_dict: + id_dict[key] = dateutil.parser.parse(id_dict[key]) + + return self.controller.challenge(parsed_args['ids']) + + +g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') +g.api.add_resource(ArticleAPI, '/article/<int:obj_id>', + endpoint='article.json') +g.api.add_resource(ArticlesAPI, '/articles', endpoint='articles.json') +g.api.add_resource(ArticlesChallenge, '/articles/challenge', + endpoint='articles_challenge.json') diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py new file mode 100644 index 00000000..4f703712 --- /dev/null +++ b/pyaggr3g470r/views/api/common.py @@ -0,0 +1,217 @@ +"""For a given resources, classes in the module intend to create the following +routes : + GET resource/<id> + -> to retreive one + POST resource + -> to create one + PUT resource/<id> + -> to update one + DELETE resource/<id> + -> to delete one + + GET resources + -> to retreive several + POST resources + -> to create several + PUT resources + -> to update several + DELETE resources + -> to delete several +""" +import json +import logging +import dateutil.parser +from copy import deepcopy +from functools import wraps +from werkzeug.exceptions import Unauthorized, BadRequest +from flask import request, g, session, Response +from flask.ext.restful import Resource, reqparse + +from pyaggr3g470r.lib.utils import default_handler +from pyaggr3g470r.models import User + +logger = logging.getLogger(__name__) + + +def authenticate(func): + """ + Decorator for the authentication to the web services. + """ + @wraps(func) + def wrapper(*args, **kwargs): + logged_in = False + if not getattr(func, 'authenticated', True): + logged_in = True + # authentication based on the session (already logged on the site) + elif 'email' in session or g.user.is_authenticated(): + logged_in = True + else: + # authentication via HTTP only + auth = request.authorization + user = User.query.filter(User.nickname == auth.username).first() + if user and user.check_password(auth.password) \ + and user.activation_key == "": + g.user = user + logged_in = True + + if logged_in: + return func(*args, **kwargs) + raise Unauthorized({'WWWAuthenticate': 'Basic realm="Login Required"'}) + return wrapper + + +def to_response(func): + """Will cast results of func as a result, and try to extract + a status_code for the Response object""" + def wrapper(*args, **kwargs): + status_code = 200 + result = func(*args, **kwargs) + if isinstance(result, Response): + return result + elif isinstance(result, tuple): + result, status_code = result + return Response(json.dumps(result, default=default_handler), + status=status_code) + return wrapper + + +class PyAggAbstractResource(Resource): + method_decorators = [authenticate, to_response] + attrs = {} + to_date = [] # list of fields to cast to datetime + + def __init__(self, *args, **kwargs): + super(PyAggAbstractResource, self).__init__(*args, **kwargs) + + @property + def controller(self): + return self.controller_cls(getattr(g.user, 'id', None)) + + def reqparse_args(self, req=None, strict=False, default=True, args=None): + """ + strict: bool + if True will throw 400 error if args are defined and not in request + default: bool + if True, won't return defaults + args: dict + the args to parse, if None, self.attrs will be used + """ + parser = reqparse.RequestParser() + for attr_name, attrs in (args or self.attrs).items(): + if attrs.pop('force_default', False): + parser.add_argument(attr_name, location='json', **attrs) + elif not default and (not request.json + or request.json and attr_name not in request.json): + continue + else: + parser.add_argument(attr_name, location='json', **attrs) + parsed = parser.parse_args(strict=strict) if req is None \ + else parser.parse_args(req, strict=strict) + for field in self.to_date: + if parsed.get(field): + try: + parsed[field] = dateutil.parser.parse(parsed[field]) + except Exception: + logger.exception('failed to parse %r', parsed[field]) + return parsed + + +class PyAggResourceNew(PyAggAbstractResource): + + def post(self): + """Create a single new object""" + return self.controller.create(**self.reqparse_args()), 201 + + +class PyAggResourceExisting(PyAggAbstractResource): + + def get(self, obj_id=None): + """Retreive a single object""" + return self.controller.get(id=obj_id) + + def put(self, obj_id=None): + """update an object, new attrs should be passed in the payload""" + args = self.reqparse_args(default=False) + new_values = {key: args[key] for key in + set(args).intersection(self.attrs)} + self.controller.update({'id': obj_id}, new_values) + + def delete(self, obj_id=None): + """delete a object""" + self.controller.delete(obj_id) + return None, 204 + + +class PyAggResourceMulti(PyAggAbstractResource): + + def get(self): + """retreive several objects. filters can be set in the payload on the + different fields of the object, and a limit can be set in there as well + """ + if 'application/json' != request.headers.get('Content-Type'): + raise BadRequest("Content-Type must be application/json") + limit = request.json.pop('limit', 10) + if not limit: + return [res for res in self.controller.read(**request.json).all()] + return [res for res in self.controller.read(**request.json).limit(limit)] + + def post(self): + """creating several objects. payload should be a list of dict. + """ + if 'application/json' != request.headers.get('Content-Type'): + raise BadRequest("Content-Type must be application/json") + status = 201 + results = [] + for attrs in request.json: + try: + results.append(self.controller.create(**attrs).id) + except Exception as error: + status = 206 + results.append(str(error)) + # if no operation succeded, it's not partial anymore, returning err 500 + if status == 206 and results.count('ok') == 0: + status = 500 + return results, status + + def put(self): + """creating several objects. payload should be: + >>> payload + [[obj_id1, {attr1: val1, attr2: val2}] + [obj_id2, {attr1: val1, attr2: val2}]] + """ + if 'application/json' != request.headers.get('Content-Type'): + raise BadRequest("Content-Type must be application/json") + status = 200 + results = [] + for obj_id, attrs in request.json: + try: + new_values = {key: attrs[key] for key in + set(attrs).intersection(self.attrs)} + self.controller.update({'id': obj_id}, new_values) + results.append('ok') + except Exception as error: + status = 206 + results.append(str(error)) + # if no operation succeded, it's not partial anymore, returning err 500 + if status == 206 and results.count('ok') == 0: + status = 500 + return results, status + + def delete(self): + """will delete several objects, + a list of their ids should be in the payload""" + if 'application/json' != request.headers.get('Content-Type'): + raise BadRequest("Content-Type must be application/json") + status = 204 + results = [] + for obj_id in request.json: + try: + self.controller.delete(obj_id) + results.append('ok') + except Exception as error: + status = 206 + results.append(error) + # if no operation succeded, it's not partial anymore, returning err 500 + if status == 206 and results.count('ok') == 0: + status = 500 + return results, status diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py new file mode 100644 index 00000000..898e30b0 --- /dev/null +++ b/pyaggr3g470r/views/api/feed.py @@ -0,0 +1,57 @@ +from flask import g + +from pyaggr3g470r.controllers.feed import FeedController, \ + DEFAULT_MAX_ERROR, DEFAULT_LIMIT + +from pyaggr3g470r.views.api.common import PyAggAbstractResource, \ + PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti + + +FEED_ATTRS = {'title': {'type': str}, + 'description': {'type': str}, + 'link': {'type': str}, + 'site_link': {'type': str}, + 'enabled': {'type': bool, 'default': True}, + 'etag': {'type': str, 'default': ''}, + 'last_modified': {'type': str}, + 'last_retreived': {'type': str}, + 'last_error': {'type': str}, + 'error_count': {'type': int, 'default': 0}} + + +class FeedNewAPI(PyAggResourceNew): + controller_cls = FeedController + attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] + + +class FeedAPI(PyAggResourceExisting): + controller_cls = FeedController + attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] + + +class FeedsAPI(PyAggResourceMulti): + controller_cls = FeedController + attrs = FEED_ATTRS + to_date = ['date', 'last_retreived'] + + +class FetchableFeedAPI(PyAggAbstractResource): + controller_cls = FeedController + to_date = ['date', 'last_retreived'] + attrs = {'max_error': {'type': int, 'default': DEFAULT_MAX_ERROR}, + 'limit': {'type': int, 'default': DEFAULT_LIMIT}} + + def get(self): + return [feed for feed in self.controller.list_fetchable( + **self.reqparse_args())] + + +g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') +g.api.add_resource(FeedAPI, '/feed/<int:obj_id>', endpoint='feed.json') +g.api.add_resource(FeedsAPI, '/feeds', endpoint='feeds.json') +g.api.add_resource(FetchableFeedAPI, '/feeds/fetchable', + endpoint='fetchable_feed.json') diff --git a/pyaggr3g470r/views/article.py b/pyaggr3g470r/views/article.py new file mode 100644 index 00000000..66cc0f37 --- /dev/null +++ b/pyaggr3g470r/views/article.py @@ -0,0 +1,53 @@ +from flask import Blueprint, g, render_template, redirect +from sqlalchemy import desc + +from pyaggr3g470r import controllers, utils +from pyaggr3g470r.decorators import pyagg_default_decorator + +articles_bp = Blueprint('articles', __name__, url_prefix='/articles') +article_bp = Blueprint('article', __name__, url_prefix='/article') + + +@articles_bp.route('/<feed_id>', methods=['GET']) +@articles_bp.route('/<feed_id>/<int:nb_articles>', methods=['GET']) +@pyagg_default_decorator +def articles(feed_id=None, nb_articles=-1): + """List articles of a feed. The administrator of the platform is able to + access to this view for every users.""" + feed = controllers.FeedController(g.user.id).get(id=feed_id) + feed.articles = controllers.ArticleController(g.user.id)\ + .read(feed_id=feed.id)\ + .order_by(desc("Article.date")) + if len(feed.articles.all()) <= nb_articles: + nb_articles = -1 + if nb_articles == -1: + feed.articles = feed.articles.limit(nb_articles) + return render_template('articles.html', feed=feed, nb_articles=nb_articles) + + +@article_bp.route('/redirect/<int:article_id>', methods=['GET']) +@pyagg_default_decorator +def redirect_to_article(article_id): + article = controllers.ArticleController(g.user.id).get(id=article_id) + return redirect(article.link) + + +@article_bp.route('/<int:article_id>', methods=['GET']) +@pyagg_default_decorator +def article(article_id=None): + """ + Presents the content of an article. + """ + article = controllers.ArticleController(g.user.id).get(id=article_id) + previous_article = article.previous_article() + if previous_article is None: + previous_article = article.source.articles[0] + next_article = article.next_article() + if next_article is None: + next_article = article.source.articles[-1] + + return render_template('article.html', + head_title=utils.clear_string(article.title), + article=article, + previous_article=previous_article, + next_article=next_article) diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py new file mode 100644 index 00000000..2af502a7 --- /dev/null +++ b/pyaggr3g470r/views/feed.py @@ -0,0 +1,50 @@ +from datetime import datetime +from flask import Blueprint, g, render_template + +from pyaggr3g470r import controllers, utils +from pyaggr3g470r.decorators import pyagg_default_decorator, \ + feed_access_required + +feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds') +feed_bp = Blueprint('feed', __name__, url_prefix='/feed') + +@feeds_bp.route('/', methods=['GET']) +def feeds(): + "Lists the subscribed feeds in a table." + return render_template('feeds.html', + feeds=controllers.FeedController(g.user.id).read()) + + +@feed_bp.route('/<int:feed_id>', methods=['GET']) +@pyagg_default_decorator +@feed_access_required +def feed(feed_id=None): + "Presents detailed information about a feed." + feed = controllers.FeedController(g.user.id).get(id=feed_id) + word_size = 6 + articles = controllers.ArticleController(g.user.id)\ + .read(feed_id=feed_id).all() + nb_articles = controllers.ArticleController(g.user.id).read().count() + top_words = utils.top_words(articles, n=50, size=int(word_size)) + tag_cloud = utils.tag_cloud(top_words) + + today = datetime.now() + try: + last_article = articles[0].date + first_article = articles[-1].date + delta = last_article - first_article + average = round(float(len(articles)) / abs(delta.days), 2) + except: + last_article = datetime.fromtimestamp(0) + first_article = datetime.fromtimestamp(0) + delta = last_article - first_article + average = 0 + elapsed = today - last_article + + return render_template('feed.html', + head_title=utils.clear_string(feed.title), + feed=feed, tag_cloud=tag_cloud, + first_post_date=first_article, + end_post_date=last_article, + nb_articles=nb_articles, + average=average, delta=delta, elapsed=elapsed) diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views/views.py index e5e07cde..d42d5db8 100644 --- a/pyaggr3g470r/views.py +++ b/pyaggr3g470r/views/views.py @@ -27,9 +27,14 @@ __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "AGPLv3" import os +import json +import string +import random +import hashlib import datetime from collections import namedtuple -from flask import abort, render_template, request, flash, session, \ +from bootstrap import application as app, db +from flask import render_template, request, flash, session, Response, \ url_for, redirect, g, current_app, make_response, jsonify from flask.ext.login import LoginManager, login_user, logout_user, \ login_required, current_user, AnonymousUserMixin @@ -37,17 +42,17 @@ from flask.ext.principal import Principal, Identity, AnonymousIdentity, \ identity_changed, identity_loaded, Permission,\ RoleNeed, UserNeed from flask.ext.babel import gettext -from sqlalchemy import desc, func, or_ +from sqlalchemy import func, or_ from sqlalchemy.exc import IntegrityError from werkzeug import generate_password_hash import conf from pyaggr3g470r import utils, notifications, export, duplicate -from pyaggr3g470r import app, db, allowed_file, babel from pyaggr3g470r.models import User, Feed, Article, Role from pyaggr3g470r.decorators import feed_access_required from pyaggr3g470r.forms import SignupForm, SigninForm, AddFeedForm, \ ProfileForm, InformationMessageForm, RecoverPasswordForm +from pyaggr3g470r.controllers import FeedController if not conf.ON_HEROKU: import pyaggr3g470r.search as fastsearch @@ -118,7 +123,7 @@ def redirect_url(default='home'): request.referrer or \ url_for(default) -@babel.localeselector +@g.babel.localeselector def get_locale(): """ Called before each request to give us a chance to choose @@ -126,7 +131,7 @@ def get_locale(): """ return request.accept_languages.best_match(conf.LANGUAGES.keys()) -@babel.timezoneselector +@g.babel.timezoneselector def get_timezone(): try: return conf.TIME_ZONE[get_locale()] @@ -152,11 +157,13 @@ def login(): login_user(user) g.user = user session['email'] = form.email.data - identity_changed.send(current_app._get_current_object(), identity=Identity(user.id)) + identity_changed.send(current_app._get_current_object(), + identity=Identity(user.id)) flash(gettext("Logged in successfully."), 'success') return redirect(url_for('home')) return render_template('login.html', form=form) + @app.route('/logout') @login_required def logout(): @@ -240,23 +247,14 @@ def home(): unread = db.session.query(Article.feed_id, func.count(Article.id))\ .filter(Article.readed == False, Article.user_id == g.user.id)\ .group_by(Article.feed_id).all() + in_error = {feed.id: feed.error_count for feed in + FeedController(g.user.id).read(error_count__gt=0).all()} def gen_url(filter_=filter_, limit=limit, feed=feed_id): return '?filter_=%s&limit=%s&feed=%d' % (filter_, limit, feed) return render_template('home.html', gen_url=gen_url, feed_id=feed_id, filter_=filter_, limit=limit, feeds=feeds, - unread=dict(unread), articles=articles.all()) - - -@app.route('/article/redirect/<int:article_id>', methods=['GET']) -@login_required -def redirect_to_article(article_id): - article = Article.query.filter(Article.id == article_id, - Article.user_id == g.user.id).first() - if article is None: - abort(404) - article.readed = True - db.session.commit() - return redirect(article.link) + unread=dict(unread), articles=articles.all(), + in_error=in_error) @app.route('/fetch', methods=['GET']) @@ -278,71 +276,6 @@ def about(): """ return render_template('about.html') -@app.route('/feeds', methods=['GET']) -@login_required -def feeds(): - """ - Lists the subscribed feeds in a table. - """ - user = User.query.filter(User.email == g.user.email).first() - return render_template('feeds.html', feeds=user.feeds) - -@app.route('/feed/<int:feed_id>', methods=['GET']) -@login_required -@feed_access_required -def feed(feed_id=None): - """ - Presents detailed information about a feed. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - word_size = 6 - articles = feed.articles.all() - nb_articles = len(Article.query.filter(Article.user_id == g.user.id).all()) - top_words = utils.top_words(articles, n=50, size=int(word_size)) - tag_cloud = utils.tag_cloud(top_words) - - today = datetime.datetime.now() - try: - last_article = articles[0].date - first_article = articles[-1].date - delta = last_article - first_article - average = round(float(len(articles)) / abs(delta.days), 2) - except: - last_article = datetime.datetime.fromtimestamp(0) - first_article = datetime.datetime.fromtimestamp(0) - delta = last_article - first_article - average = 0 - elapsed = today - last_article - - return render_template('feed.html', head_title=utils.clear_string(feed.title), feed=feed, tag_cloud=tag_cloud, \ - first_post_date=first_article, end_post_date=last_article , nb_articles=nb_articles, \ - average=average, delta=delta, elapsed=elapsed) - -@app.route('/article/<int:article_id>', methods=['GET']) -@login_required -def article(article_id=None): - """ - Presents the content of an article. - """ - article = Article.query.filter(Article.user_id == g.user.id, Article.id == article_id).first() - if article is not None: - if not article.readed: - article.readed = True - db.session.commit() - - previous_article = article.previous_article() - if previous_article is None: - previous_article = article.source.articles[0] - next_article = article.next_article() - if next_article is None: - next_article = article.source.articles[-1] - - return render_template('article.html', head_title=utils.clear_string(article.title), - article=article, - previous_article=previous_article, next_article=next_article) - flash(gettext("This article do not exist."), 'warning') - return redirect(url_for('home')) - @app.route('/mark_as/<string:new_value>', methods=['GET']) @app.route('/mark_as/<string:new_value>/feed/<int:feed_id>', methods=['GET']) @@ -403,24 +336,6 @@ def delete(article_id=None): flash(gettext('This article do not exist.'), 'danger') return redirect(url_for('home')) -@app.route('/articles/<feed_id>', methods=['GET']) -@app.route('/articles/<feed_id>/<int:nb_articles>', methods=['GET']) -@login_required -@feed_access_required -def articles(feed_id=None, nb_articles=-1): - """ - List articles of a feed. - The administrator of the platform is able to access to this view for every users. - """ - feed = Feed.query.filter(Feed.id == feed_id).first() - new_feed = feed - if len(feed.articles.all()) <= nb_articles: - nb_articles = -1 - if nb_articles == -1: - nb_articles = int(1e9) - new_feed.articles = Article.query.filter(Article.user_id == g.user.id, \ - Article.feed_id == feed.id).order_by(desc("Article.date")).limit(nb_articles) - return render_template('articles.html', feed=new_feed, nb_articles=nb_articles) @app.route('/favorites', methods=['GET']) @login_required @@ -591,7 +506,7 @@ def management(): if None != request.files.get('opmlfile', None): # Import an OPML file data = request.files.get('opmlfile', None) - if not allowed_file(data.filename): + if not g.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: @@ -604,7 +519,7 @@ def management(): elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) - if not allowed_file(data.filename): + if not g.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: @@ -627,7 +542,7 @@ def management(): @app.route('/history', methods=['GET']) @login_required def history(): - user = User.query.filter(User.id == g.user.id).first() + #user = User.query.filter(User.id == g.user.id).first() return render_template('history.html') @app.route('/bookmarklet', methods=['GET']) @@ -653,7 +568,7 @@ def edit_feed(feed_id=None): return redirect('/edit_feed/' + str(feed_id)) else: # Create a new feed - existing_feed = [feed for feed in g.user.feeds if feed.link == form.link.data] + existing_feed = [f for f in g.user.feeds if feed.link == form.link.data] if len(existing_feed) == 0: new_feed = Feed(title=form.title.data, description="", link=form.link.data, \ site_link=form.site_link.data, enabled=form.enabled.data) @@ -678,7 +593,7 @@ def edit_feed(feed_id=None): # Enable the user to add a feed with a bookmarklet if None is not request.args.get('url', None): - existing_feed = [feed for feed in g.user.feeds if feed.link == request.args.get('url', None)] + existing_feed = [f for f in g.user.feeds if feed.link == request.args.get('url', None)] if len(existing_feed) == 0: g.user.feeds.append(Feed(link=request.args.get('url', None))) db.session.commit() @@ -777,8 +692,6 @@ def recover(): Enables the user to recover its account when he has forgotten its password. """ - import string - import random form = RecoverPasswordForm() if request.method == 'POST': @@ -920,7 +833,6 @@ def disable_user(user_id=None): flash(gettext('Problem while sending activation email') + ': ' + str(e), 'danger') else: - import random, hashlib user.activation_key = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()[:86] flash(gettext('Account of the user') + ' ' + user.nickname + ' ' + gettext('successfully disabled.'), 'success') db.session.commit() diff --git a/requirements.txt b/requirements.txt index 4f9a082f..c988f7dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ requests beautifulsoup4 lxml SQLAlchemy -psycopg2 Flask Flask-SQLAlchemy Flask-Login @@ -14,10 +13,11 @@ Flask-WTF Flask-RESTful Flask-Babel Flask-SSLify +Flask-Migrate +flask-Script WTForms python-postmark whoosh python-dateutil alembic -flask-Migrate -flask-Script +requests-futures==0.9.5 diff --git a/runserver.py b/runserver.py index 8ae7282a..2ced409f 100755 --- a/runserver.py +++ b/runserver.py @@ -19,8 +19,43 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from bootstrap import conf -from pyaggr3g470r import app as application +from bootstrap import conf, application, db, populate_g +from flask.ext.babel import Babel +from flask.ext.babel import format_datetime + +if conf.ON_HEROKU: + from flask_sslify import SSLify + SSLify(application) + +ALLOWED_EXTENSIONS = set(['xml', 'opml', 'json']) + +def allowed_file(filename): + """ + Check if the uploaded file is allowed. + """ + return '.' in filename and \ + filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS + +babel = Babel(application) + +application.jinja_env.filters['datetime'] = format_datetime + +# Views +from flask.ext.restful import Api +from flask import g + +with application.app_context(): + populate_g() + g.api = Api(application, prefix='/api/v2.0') + g.babel = babel + g.allowed_file = allowed_file + + from pyaggr3g470r import views + application.register_blueprint(views.articles_bp) + application.register_blueprint(views.article_bp) + application.register_blueprint(views.feeds_bp) + application.register_blueprint(views.feed_bp) + if __name__ == '__main__': application.run(host=conf.WEBSERVER_HOST, |