From 83081ad7e488c44757e43ff40e83458a2e1451ed Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Tue, 1 Mar 2016 22:47:53 +0100 Subject: begin integration of the new architecture --- .../2472eddbf44b_update_of_the_user_model.py | 39 +++ src/conf.py | 3 + src/manager.py | 2 +- src/web/controllers/abstract.py | 46 ++- src/web/controllers/article.py | 18 +- src/web/controllers/feed.py | 28 +- src/web/controllers/user.py | 5 +- src/web/forms.py | 45 ++- src/web/lib/feed_utils.py | 14 + src/web/lib/utils.py | 4 +- src/web/models/__init__.py | 8 +- src/web/models/article.py | 18 +- src/web/models/category.py | 16 +- src/web/models/feed.py | 22 +- src/web/models/right_mixin.py | 54 +++ src/web/models/user.py | 16 +- src/web/templates/layout.html | 6 +- src/web/views/__init__.py | 23 +- src/web/views/admin.py | 32 +- src/web/views/article.py | 16 +- src/web/views/category.py | 14 +- src/web/views/common.py | 53 +++ src/web/views/feed.py | 36 +- src/web/views/home.py | 155 +++++++++ src/web/views/session_mgmt.py | 123 +++++++ src/web/views/user.py | 22 +- src/web/views/views.py | 386 +-------------------- 27 files changed, 665 insertions(+), 539 deletions(-) create mode 100644 migrations/versions/2472eddbf44b_update_of_the_user_model.py create mode 100644 src/web/models/right_mixin.py create mode 100644 src/web/views/common.py create mode 100644 src/web/views/home.py create mode 100644 src/web/views/session_mgmt.py diff --git a/migrations/versions/2472eddbf44b_update_of_the_user_model.py b/migrations/versions/2472eddbf44b_update_of_the_user_model.py new file mode 100644 index 00000000..e0974015 --- /dev/null +++ b/migrations/versions/2472eddbf44b_update_of_the_user_model.py @@ -0,0 +1,39 @@ +"""update of the user model + +Revision ID: 2472eddbf44b +Revises: ac35c979311a +Create Date: 2016-03-01 22:35:03.659694 + +""" + +# revision identifiers, used by Alembic. +revision = '2472eddbf44b' +down_revision = 'ac35c979311a' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.drop_column('user', 'roles') + op.drop_column('user', 'enabled') + op.add_column('user', sa.Column('is_active', sa.Boolean(), default=False)) + op.add_column('user', sa.Column('is_admin', sa.Boolean(), default=False)) + op.add_column('user', sa.Column('is_api', sa.Boolean(), default=False)) + op.drop_table('role') + + + +def downgrade(): + op.drop_column('user', 'is_active') + op.drop_column('user', 'is_admin') + op.drop_column('user', 'is_api') + op.create_table('role', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(), nullable=True), + sa.Column('user_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name')) diff --git a/src/conf.py b/src/conf.py index 0fcb330e..9e7f1d13 100644 --- a/src/conf.py +++ b/src/conf.py @@ -9,6 +9,7 @@ import logging BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PATH = os.path.abspath(".") +API_ROOT = '/api/v2.0' # available languages LANGUAGES = { @@ -23,6 +24,7 @@ TIME_ZONE = { ON_HEROKU = int(os.environ.get('HEROKU', 0)) == 1 DEFAULTS = {"platform_url": "https://jarr.herokuapp.com/", + "self_registration": "false", "cdn_address": "", "admin_email": "root@jarr.localhost", "postmark_api_key": "", @@ -76,6 +78,7 @@ else: PLATFORM_URL = config.get('misc', 'platform_url') ADMIN_EMAIL = config.get('misc', 'admin_email') +SELF_REGISTRATION = config.getboolean('misc', 'self_registration') RECAPTCHA_PUBLIC_KEY = config.get('misc', 'recaptcha_public_key') RECAPTCHA_PRIVATE_KEY = config.get('misc', 'recaptcha_private_key') diff --git a/src/manager.py b/src/manager.py index e64263f2..14139ec2 100755 --- a/src/manager.py +++ b/src/manager.py @@ -64,7 +64,7 @@ def fetch_asyncio(user_id, feed_id): loop = asyncio.get_event_loop() for user in users: - if user.enabled: + if user.is_active: print("Fetching articles for " + user.nickname) g.user = user classic_crawler.retrieve_feed(loop, g.user, feed_id) diff --git a/src/web/controllers/abstract.py b/src/web/controllers/abstract.py index 828e6a29..2a2e6f9f 100644 --- a/src/web/controllers/abstract.py +++ b/src/web/controllers/abstract.py @@ -1,30 +1,29 @@ import logging -from flask import g +import dateutil.parser from bootstrap import db +from datetime import datetime +from collections import defaultdict from sqlalchemy import or_, func from werkzeug.exceptions import Forbidden, NotFound logger = logging.getLogger(__name__) -class AbstractController(object): +class AbstractController: _db_cls = None # reference to the database class _user_id_key = 'user_id' - def __init__(self, user_id=None): + def __init__(self, user_id=None, ignore_context=False): """User id is a right management mechanism that should be used to filter objects in database on their denormalized "user_id" field (or "id" field for users). Should no user_id be provided, the Controller won't apply any filter allowing for a kind of "super user" mode. """ - self.user_id = user_id try: - if self.user_id is not None \ - and self.user_id != g.user.id and not g.user.is_admin(): - self.user_id = g.user.id - except RuntimeError: # passing on out of context errors - pass + self.user_id = int(user_id) + except TypeError: + self.user_id = user_id def _to_filters(self, **filters): """ @@ -83,6 +82,7 @@ class AbstractController(object): return obj def create(self, **attrs): + assert attrs, "attributes to update must not be empty" if self._user_id_key is not None and self._user_id_key not in attrs: attrs[self._user_id_key] = self.user_id assert self._user_id_key is None or self._user_id_key in attrs \ @@ -98,6 +98,7 @@ class AbstractController(object): return self._get(**filters) def update(self, filters, attrs): + assert attrs, "attributes to update must not be empty" result = self._get(**filters).update(attrs, synchronize_session=False) db.session.commit() return result @@ -121,3 +122,30 @@ class AbstractController(object): return dict(db.session.query(elem_to_group_by, func.count('id')) .filter(*self._to_filters(**filters)) .group_by(elem_to_group_by).all()) + + @classmethod + def _get_attrs_desc(cls, role, right=None): + result = defaultdict(dict) + if role == 'admin': + columns = cls._db_cls.__table__.columns.keys() + else: + assert role in {'base', 'api'}, 'unknown role %r' % role + assert right in {'read', 'write'}, \ + "right must be 'read' or 'write' with role %r" % role + columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() + for column in columns: + result[column] = {} + db_col = getattr(cls._db_cls, column).property.columns[0] + try: + result[column]['type'] = db_col.type.python_type + except NotImplementedError: + if db_col.default: + result[column]['type'] = db_col.default.arg.__class__ + if column not in result: + continue + if issubclass(result[column]['type'], datetime): + result[column]['default'] = datetime.utcnow() + result[column]['type'] = lambda x: dateutil.parser.parse(x) + elif db_col.default: + result[column]['default'] = db_col.default.arg + return result diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index bc9ef36e..8c6952cb 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -6,7 +6,7 @@ from collections import Counter from bootstrap import db from .abstract import AbstractController -from web.controllers import FeedController +from web.controllers import CategoryController, FeedController from web.models import Article logger = logging.getLogger(__name__) @@ -35,11 +35,12 @@ class ArticleController(AbstractController): def create(self, **attrs): # handling special denorm for article rights - assert 'feed_id' in attrs + assert 'feed_id' in attrs, "must provide feed_id when creating article" feed = FeedController( attrs.get('user_id', self.user_id)).get(id=attrs['feed_id']) if 'user_id' in attrs: - assert feed.user_id == attrs['user_id'] or self.user_id is None + assert feed.user_id == attrs['user_id'] or self.user_id is None, \ + "no right on feed %r" % feed.id attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id # handling feed's filters @@ -66,6 +67,17 @@ class ArticleController(AbstractController): return super().create(**attrs) + def update(self, filters, attrs): + user_id = attrs.get('user_id', self.user_id) + if 'feed_id' in attrs: + feed = FeedController().get(id=attrs['feed_id']) + assert feed.user_id == user_id, "no right on feed %r" % feed.id + attrs['category_id'] = feed.category_id + if attrs.get('category_id'): + cat = CategoryController().get(id=attrs['category_id']) + assert cat.user_id == user_id, "no right on cat %r" % cat.id + return super().update(filters, attrs) + def get_history(self, year=None, month=None): """ Sort articles by year and month. diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 78caf2e1..95b1eceb 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -1,24 +1,3 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# jarr - A Web based news aggregator. -# Copyright (C) 2010-2016 Cédric Bonhomme - https://www.cedricbonhomme.org -# -# For more information : https://github.com/JARR-aggregator/JARR/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - import logging import itertools from datetime import datetime, timedelta @@ -26,7 +5,7 @@ from datetime import datetime, timedelta import conf from .abstract import AbstractController from .icon import IconController -from web.models import Feed +from web.models import User, Feed from web.lib.utils import clear_string logger = logging.getLogger(__name__) @@ -43,11 +22,12 @@ class FeedController(AbstractController): return [feed for feed in self.read( error_count__lt=max_error, enabled=True, last_retrieved__lt=max_last) + .join(User).filter(User.is_active == True) .order_by('last_retrieved') .limit(limit)] - def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, limit=DEFAULT_LIMIT, - refresh_rate=DEFAULT_REFRESH_RATE): + def list_fetchable(self, max_error=DEFAULT_MAX_ERROR, + limit=DEFAULT_LIMIT, refresh_rate=DEFAULT_REFRESH_RATE): now = datetime.now() max_last = now - timedelta(minutes=refresh_rate) feeds = self.list_late(max_last, max_error, limit) diff --git a/src/web/controllers/user.py b/src/web/controllers/user.py index ae169b05..ee2eb4c2 100644 --- a/src/web/controllers/user.py +++ b/src/web/controllers/user.py @@ -1,6 +1,6 @@ import random import hashlib -from werkzeug import generate_password_hash +from werkzeug import generate_password_hash, check_password_hash from .abstract import AbstractController from web.models import User @@ -15,6 +15,9 @@ class UserController(AbstractController): elif 'password' in attrs: del attrs['password'] + def check_password(self, user, password): + return check_password_hash(user.pwdhash, password) + def create(self, **attrs): self._handle_password(attrs) return super().create(**attrs) diff --git a/src/web/forms.py b/src/web/forms.py index b17d2f7a..b23bf77f 100644 --- a/src/web/forms.py +++ b/src/web/forms.py @@ -30,12 +30,14 @@ __license__ = "GPLv3" from flask import flash, url_for, redirect from flask.ext.wtf import Form from flask.ext.babel import lazy_gettext +from werkzeug.exceptions import NotFound from wtforms import TextField, TextAreaField, PasswordField, BooleanField, \ SubmitField, IntegerField, SelectField, validators, HiddenField from flask.ext.wtf.html5 import EmailField from flask_wtf import RecaptchaField from web import utils +from web.controllers import UserController from web.models import User @@ -52,15 +54,16 @@ class SignupForm(Form): password = PasswordField(lazy_gettext("Password"), [validators.Required(lazy_gettext("Please enter a password.")), validators.Length(min=6, max=100)]) - recaptcha = RecaptchaField() submit = SubmitField(lazy_gettext("Sign up")) def validate(self): - validated = super(SignupForm, self).validate() - if self.nickname.data != User.make_valid_nickname(self.nickname.data): - self.nickname.errors.append(lazy_gettext( - 'This nickname has invalid characters. ' - 'Please use letters, numbers, dots and underscores only.')) + ucontr = UserController() + validated = super().validate() + if ucontr.read(login=self.login.data).count(): + self.login.errors.append('Login already taken') + validated = False + if self.password.data != self.password_conf.data: + self.password_conf.errors.append("Passwords don't match") validated = False return validated @@ -94,19 +97,27 @@ class SigninForm(RedirectForm): validators.Length(min=6, max=100)]) submit = SubmitField(lazy_gettext("Log In")) - def validate(self): - if not super(SigninForm, self).validate(): - return False + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.user = None - user = User.query.filter(User.email == self.email.data).first() - if user and user.check_password(self.password.data) and user.enabled: - return True - elif user and not user.enabled: - flash(lazy_gettext('Account not confirmed'), 'danger') - return False + def validate(self): + validated = super().validate() + ucontr = UserController() + try: + user = ucontr.get(email=self.email.data) + except NotFound: + self.login.errors.append('Wrong login') + validated = False else: - flash(lazy_gettext('Invalid email or password'), 'danger') - return False + if not user.is_active: + self.login.errors.append('User is desactivated') + validated = False + if not ucontr.check_password(user, self.password.data): + self.password.errors.append('Wrong password') + validated = False + self.user = user + return validated class UserForm(Form): diff --git a/src/web/lib/feed_utils.py b/src/web/lib/feed_utils.py index 80800bec..9925613f 100644 --- a/src/web/lib/feed_utils.py +++ b/src/web/lib/feed_utils.py @@ -1,3 +1,4 @@ +import html import urllib import logging import requests @@ -17,6 +18,19 @@ def is_parsing_ok(parsed_feed): return parsed_feed['entries'] or not parsed_feed['bozo'] +def escape_keys(*keys): + def wrapper(func): + def metawrapper(*args, **kwargs): + result = func(*args, **kwargs) + for key in keys: + if key in result: + result[key] = html.unescape(result[key]) + return result + return metawrapper + return wrapper + + +@escape_keys('title', 'description') def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): requests_kwargs = {'headers': {'User-Agent': USER_AGENT}, 'verify': False} if url is None and fp_parsed is not None: diff --git a/src/web/lib/utils.py b/src/web/lib/utils.py index 88d24ba5..f2bed3ff 100644 --- a/src/web/lib/utils.py +++ b/src/web/lib/utils.py @@ -9,12 +9,12 @@ from flask import request, url_for logger = logging.getLogger(__name__) -def default_handler(obj): +def default_handler(obj, role='admin'): """JSON handler for default query formatting""" if hasattr(obj, 'isoformat'): return obj.isoformat() if hasattr(obj, 'dump'): - return obj.dump() + return obj.dump(role=role) if isinstance(obj, (set, frozenset, types.GeneratorType)): return list(obj) if isinstance(obj, BaseException): diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py index d9489dbb..7c50cab3 100644 --- a/src/web/models/__init__.py +++ b/src/web/models/__init__.py @@ -88,17 +88,13 @@ def db_create(db): "Will create the database from conf parameters." db.create_all() - role_admin = Role(name="admin") - role_user = Role(name="user") - user1 = User(nickname="admin", email=os.environ.get("ADMIN_EMAIL", "root@jarr.localhost"), pwdhash=generate_password_hash( os.environ.get("ADMIN_PASSWORD", "password")), - enabled=True) - user1.roles.extend([role_admin, role_user]) + is_admin=True) db.session.add(user1) db.session.commit() - return role_admin, role_user + return user1 diff --git a/src/web/models/article.py b/src/web/models/article.py index 1fee7096..d3c0bed2 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -29,9 +29,10 @@ __license__ = "GPLv3" from bootstrap import db from datetime import datetime from sqlalchemy import asc, desc +from web.models.right_mixin import RightMixin -class Article(db.Model): +class Article(db.Model, RightMixin): "Represent an article from a feed." id = db.Column(db.Integer(), primary_key=True) entry_id = db.Column(db.String()) @@ -68,18 +69,3 @@ class Article(db.Model): return "" % (self.id, self.entry_id, self.title, self.date, self.retrieved_date) - - def dump(self): - return {"id": self.id, - "user_id": self.user_id, - "entry_id": self.entry_id, - "title": self.title, - "link": self.link, - "content": self.content, - "readed": self.readed, - "like": self.like, - "date": self.date, - "updated_date": self.updated_date, - "retrieved_date": self.retrieved_date, - "feed_id": self.feed_id, - "category_id": self.category_id} diff --git a/src/web/models/category.py b/src/web/models/category.py index 78054809..c35db52e 100644 --- a/src/web/models/category.py +++ b/src/web/models/category.py @@ -1,11 +1,21 @@ from bootstrap import db +from sqlalchemy import Index +from web.models.right_mixin import RightMixin -class Category(db.Model): +class Category(db.Model, RightMixin): id = db.Column(db.Integer(), primary_key=True) name = db.Column(db.String()) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - def dump(self): - return {key: getattr(self, key) for key in ('id', 'name', 'user_id')} + idx_category_uid = Index('user_id') + + # api whitelists + @staticmethod + def _fields_base_read(): + return {'id', 'user_id'} + + @staticmethod + def _fields_base_write(): + return {'name'} diff --git a/src/web/models/feed.py b/src/web/models/feed.py index cb78f7ad..ba9255e9 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -29,9 +29,10 @@ __license__ = "GPLv3" from bootstrap import db from datetime import datetime from sqlalchemy import desc +from web.models.right_mixin import RightMixin -class Feed(db.Model): +class Feed(db.Model, RightMixin): """ Represent a feed. """ @@ -63,22 +64,3 @@ class Feed(db.Model): def __repr__(self): return '' % (self.title) - - def dump(self): - return {"id": self.id, - "user_id": self.user_id, - "category_id": self.category_id, - "title": self.title, - "description": self.description, - "link": self.link, - "site_link": self.site_link, - "etag": self.etag, - "enabled": self.enabled, - "filters": self.filters, - "icon_url": self.icon_url, - "error_count": self.error_count, - "last_error": self.last_error, - "created_date": self.created_date, - "last_modified": self.last_modified, - "last_retrieved": self.last_retrieved, - "nb_articles": self.articles.count()} diff --git a/src/web/models/right_mixin.py b/src/web/models/right_mixin.py new file mode 100644 index 00000000..c4d92008 --- /dev/null +++ b/src/web/models/right_mixin.py @@ -0,0 +1,54 @@ +class RightMixin: + + @staticmethod + def _fields_base_write(): + return {} + + @staticmethod + def _fields_base_read(): + return {'id'} + + @staticmethod + def _fields_api_write(): + return {} + + @staticmethod + def _fields_api_read(): + return {'id'} + + @classmethod + def fields_base_write(cls): + return cls._fields_base_write() + + @classmethod + def fields_base_read(cls): + return cls._fields_base_write().union(cls._fields_base_read()) + + @classmethod + def fields_api_write(cls): + return cls.fields_base_write().union(cls._fields_api_write()) + + @classmethod + def fields_api_read(cls): + return cls.fields_base_read().union(cls._fields_api_read()) + + def __getitem__(self, key): + if not hasattr(self, '__dump__'): + self.__dump__ = {} + return self.__dump__.get(key) + + def __setitem__(self, key, value): + if not hasattr(self, '__dump__'): + self.__dump__ = {} + self.__dump__[key] = value + + def dump(self, role='admin'): + if role == 'admin': + dico = {k: getattr(self, k) for k in self.__table__.columns.keys()} + elif role == 'api': + dico = {k: getattr(self, k) for k in self.fields_api_read()} + else: + dico = {k: getattr(self, k) for k in self.fields_base_read()} + if hasattr(self, '__dump__'): + dico.update(self.__dump__) + return dico diff --git a/src/web/models/user.py b/src/web/models/user.py index 1a276f7e..133901a5 100644 --- a/src/web/models/user.py +++ b/src/web/models/user.py @@ -34,9 +34,10 @@ from werkzeug import check_password_hash from flask.ext.login import UserMixin from bootstrap import db +from web.models.right_mixin import RightMixin -class User(db.Model, UserMixin): +class User(db.Model, UserMixin, RightMixin): """ Represent a user. """ @@ -44,14 +45,17 @@ class User(db.Model, UserMixin): nickname = db.Column(db.String(), unique=True) email = db.Column(db.String(254), index=True, unique=True) pwdhash = db.Column(db.String()) - roles = db.relationship('Role', backref='user', lazy='dynamic') - enabled = db.Column(db.Boolean(), default=False) date_created = db.Column(db.DateTime(), default=datetime.now) last_seen = db.Column(db.DateTime(), default=datetime.now) feeds = db.relationship('Feed', backref='subscriber', lazy='dynamic', cascade='all,delete-orphan') refresh_rate = db.Column(db.Integer, default=60) # in minutes + # user rights + is_active = db.Column(db.Boolean(), default=True) + is_admin = db.Column(db.Boolean(), default=False) + is_api = db.Column(db.Boolean(), default=False) + @staticmethod def make_valid_nickname(nickname): return re.sub('[^a-zA-Z0-9_\.]', '', nickname) @@ -68,12 +72,6 @@ class User(db.Model, UserMixin): """ return check_password_hash(self.pwdhash, password) - def is_admin(self): - """ - Return True if the user has administrator rights. - """ - return "admin" in [role.name for role in self.roles] - def __eq__(self, other): return self.id == other.id diff --git a/src/web/templates/layout.html b/src/web/templates/layout.html index feb370e3..d382a6dd 100644 --- a/src/web/templates/layout.html +++ b/src/web/templates/layout.html @@ -35,7 +35,7 @@