From 462f6d3b21558ed0a283c24e0e0332eac6ccbbb3 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Fri, 11 Sep 2015 18:28:12 +0200 Subject: base modification in model for category support --- migrations/versions/3f83bfe93fc_adding_category.py | 42 ++++++++++++++++++++++ pyaggr3g470r/controllers/category.py | 6 ++++ pyaggr3g470r/models/category.py | 8 +++++ pyaggr3g470r/views/api/category.py | 31 ++++++++++++++++ src/crawler.py | 5 +-- src/manager.py | 6 +++- src/web/controllers/article.py | 3 +- src/web/lib/crawler.py | 7 ++-- src/web/models/__init__.py | 3 +- src/web/models/article.py | 11 +++--- src/web/models/feed.py | 1 + src/web/views/api/__init__.py | 4 +-- src/web/views/api/article.py | 7 ++-- src/web/views/api/feed.py | 13 +++---- 14 files changed, 119 insertions(+), 28 deletions(-) create mode 100644 migrations/versions/3f83bfe93fc_adding_category.py create mode 100644 pyaggr3g470r/controllers/category.py create mode 100644 pyaggr3g470r/models/category.py create mode 100644 pyaggr3g470r/views/api/category.py diff --git a/migrations/versions/3f83bfe93fc_adding_category.py b/migrations/versions/3f83bfe93fc_adding_category.py new file mode 100644 index 00000000..bd65a01f --- /dev/null +++ b/migrations/versions/3f83bfe93fc_adding_category.py @@ -0,0 +1,42 @@ +"""adding category + +Revision ID: 3f83bfe93fc +Revises: 25ca960a207 +Create Date: 2015-09-01 14:15:04.212563 +""" + +# revision identifiers, used by Alembic. +revision = '3f83bfe93fc' +down_revision = '25ca960a207' + +import conf +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('category', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id')) + op.add_column('article', + sa.Column('category_id', sa.Integer(), nullable=True)) + op.add_column('feed', + sa.Column('category_id', sa.Integer(), nullable=True)) + if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: + op.create_foreign_key(None, 'article', 'category', + ['category_id'], ['id']) + op.create_foreign_key(None, 'feed', 'category', + ['category_id'], ['id']) + + +def downgrade(): + if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: + op.drop_constraint(None, 'feed', type_='foreignkey') + op.drop_constraint(None, 'feed', type_='foreignkey') + op.drop_column('feed', 'category_id') + op.drop_constraint(None, 'article', type_='foreignkey') + op.drop_column('article', 'category_id') + op.drop_table('category') diff --git a/pyaggr3g470r/controllers/category.py b/pyaggr3g470r/controllers/category.py new file mode 100644 index 00000000..b6fc591c --- /dev/null +++ b/pyaggr3g470r/controllers/category.py @@ -0,0 +1,6 @@ +from .abstract import AbstractController +from pyaggr3g470r.models import Category + + +class CategoryController(AbstractController): + _db_cls = Category diff --git a/pyaggr3g470r/models/category.py b/pyaggr3g470r/models/category.py new file mode 100644 index 00000000..513227a5 --- /dev/null +++ b/pyaggr3g470r/models/category.py @@ -0,0 +1,8 @@ +from bootstrap import db + + +class Category(db.Model): + id = db.Column(db.Integer(), primary_key=True) + name = db.Column(db.String()) + + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) diff --git a/pyaggr3g470r/views/api/category.py b/pyaggr3g470r/views/api/category.py new file mode 100644 index 00000000..31f46751 --- /dev/null +++ b/pyaggr3g470r/views/api/category.py @@ -0,0 +1,31 @@ +from flask import g + +from pyaggr3g470r.controllers.category import CategoryController +from pyaggr3g470r.views.api.common import (PyAggResourceNew, + PyAggResourceExisting, + PyAggResourceMulti) + + +CAT_ATTRS = {'name': {'type': str}, + 'user_id': {'type': int}} + + +class CategoryNewAPI(PyAggResourceNew): + controller_cls = CategoryController + attrs = CAT_ATTRS + + +class CategoryAPI(PyAggResourceExisting): + controller_cls = CategoryController + attrs = CAT_ATTRS + + +class CategoriesAPI(PyAggResourceMulti): + controller_cls = CategoryController + attrs = CAT_ATTRS + + +g.api.add_resource(CategoryNewAPI, '/category', endpoint='category_new.json') +g.api.add_resource(CategoryAPI, '/category/', + endpoint='category.json') +g.api.add_resource(CategoriesAPI, '/categories', endpoint='categories.json') diff --git a/src/crawler.py b/src/crawler.py index 1a759945..0598c418 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -58,9 +58,6 @@ else: async def get(*args, **kwargs): #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False) try: - #logger.info("Fetching the feed: " + args[0]) - #response = yield from aiohttp.request('GET', *args, **kwargs) - #return (yield from response.read_and_close(decode=False)) data = feedparser.parse(args[0]) return data except Exception as e: @@ -131,7 +128,7 @@ async def insert_database(user, feed): new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) - except Exception as e: + except Exception: logger.exception("Error when inserting article in database:") continue return new_articles diff --git a/src/manager.py b/src/manager.py index e1f0878b..f7240670 100755 --- a/src/manager.py +++ b/src/manager.py @@ -12,6 +12,7 @@ Migrate(application, db) manager = Manager(application) manager.add_command('db', MigrateCommand) + @manager.command def db_empty(): "Will drop every datas stocked in db." @@ -19,6 +20,7 @@ def db_empty(): populate_g() web.models.db_empty(db) + @manager.command def db_create(): "Will create the database from conf parameters." @@ -26,6 +28,7 @@ def db_create(): populate_g() web.models.db_create(db) + @manager.command def fetch(limit=100, retreive_all=False): "Crawl the feeds with the client crawler." @@ -34,6 +37,7 @@ def fetch(limit=100, retreive_all=False): scheduler.run(limit=limit, retreive_all=retreive_all) scheduler.wait() + @manager.command def fetch_asyncio(user_id, feed_id): "Crawl the feeds with asyncio." @@ -63,7 +67,7 @@ def fetch_asyncio(user_id, feed_id): if user.activation_key == "": print("Fetching articles for " + user.nickname) g.user = user - feed_getter = crawler.retrieve_feed(loop, g.user, feed_id) + crawler.retrieve_feed(loop, g.user, feed_id) loop.close() from scripts.probes import ArticleProbe, FeedProbe diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index 8b6926b7..a8788f46 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -34,8 +34,7 @@ class ArticleController(AbstractController): .group_by(Article.feed_id).all()) def count_by_user_id(self, **filters): - return dict(db.session.query(Article.user_id, - func.count(Article.id)) + return dict(db.session.query(Article.user_id, func.count(Article.id)) .filter(*self._to_filters(**filters)) .group_by(Article.user_id).all()) diff --git a/src/web/lib/crawler.py b/src/web/lib/crawler.py index 7343ea4d..f480fe96 100644 --- a/src/web/lib/crawler.py +++ b/src/web/lib/crawler.py @@ -18,7 +18,6 @@ import json import logging import feedparser from datetime import datetime, timedelta -from functools import wraps from time import strftime, gmtime from concurrent.futures import ThreadPoolExecutor from requests_futures.sessions import FuturesSession @@ -132,7 +131,7 @@ class PyAggUpdater(AbstractCrawler): {key: "%s -> %s" % (up_feed[key], self.feed.get(key)) for key in up_feed if up_feed[key] != self.feed.get(key)}) - future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed) class FeedCrawler(AbstractCrawler): @@ -144,8 +143,8 @@ class FeedCrawler(AbstractCrawler): def clean_feed(self): """Will reset the errors counters on a feed that have known errors""" if self.feed.get('error_count') or self.feed.get('last_error'): - future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], - {'error_count': 0, 'last_error': ''}) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': 0, 'last_error': ''}) def callback(self, response): """will fetch the feed and interprete results (304, etag) or will diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py index 54168279..81ebe361 100644 --- a/src/web/models/__init__.py +++ b/src/web/models/__init__.py @@ -31,8 +31,9 @@ from .role import Role from .user import User from .article import Article from .icon import Icon +from .category import Category -__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon'] +__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category'] import os diff --git a/src/web/models/article.py b/src/web/models/article.py index 54eefe75..44a4234d 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -32,10 +32,8 @@ from sqlalchemy import asc, desc class Article(db.Model): - """ - Represent an article from a feed. - """ - id = db.Column(db.Integer, primary_key=True) + "Represent an article from a feed." + id = db.Column(db.Integer(), primary_key=True) entry_id = db.Column(db.String()) link = db.Column(db.String()) title = db.Column(db.String()) @@ -46,8 +44,9 @@ class Article(db.Model): date = db.Column(db.DateTime(), default=datetime.now) retrieved_date = db.Column(db.DateTime(), default=datetime.now) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) + user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) + feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id')) + category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) def previous_article(self): """ diff --git a/src/web/models/feed.py b/src/web/models/feed.py index 59456a7f..87616e02 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -56,6 +56,7 @@ class Feed(db.Model): # relationship icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None) user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) + category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) articles = db.relationship('Article', backref='source', lazy='dynamic', cascade='all,delete-orphan', order_by=desc("Article.date")) diff --git a/src/web/views/api/__init__.py b/src/web/views/api/__init__.py index 24472ebe..ed3bd702 100644 --- a/src/web/views/api/__init__.py +++ b/src/web/views/api/__init__.py @@ -26,6 +26,6 @@ __revision__ = "$Date: 2014/07/05 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "AGPLv3" -from web.views.api import article, feed +from web.views.api import article, feed, category -__all__ = ['article', 'feed'] +__all__ = ['article', 'feed', 'category'] diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py index 51844b20..23c5c495 100644 --- a/src/web/views/api/article.py +++ b/src/web/views/api/article.py @@ -13,12 +13,15 @@ from web.views.api.common import PyAggAbstractResource,\ ARTICLE_ATTRS = {'user_id': {'type': int}, 'feed_id': {'type': int}, + 'category_id': {'type': int}, 'entry_id': {'type': str}, 'link': {'type': str}, 'title': {'type': str}, - 'readed': {'type': bool}, 'like': {'type': bool}, + 'readed': {'type': bool}, + 'like': {'type': bool}, 'content': {'type': str}, - 'date': {'type': str}, 'retrieved_date': {'type': str}} + 'date': {'type': str}, + 'retrieved_date': {'type': str}} class ArticleNewAPI(PyAggResourceNew): diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py index 2bb9814f..dd9919bf 100644 --- a/src/web/views/api/feed.py +++ b/src/web/views/api/feed.py @@ -4,19 +4,20 @@ from flask import g from web.controllers.feed import (FeedController, - DEFAULT_MAX_ERROR, - DEFAULT_LIMIT, - DEFAULT_REFRESH_RATE) + DEFAULT_MAX_ERROR, + DEFAULT_LIMIT, + DEFAULT_REFRESH_RATE) from web.views.api.common import PyAggAbstractResource, \ - PyAggResourceNew, \ - PyAggResourceExisting, \ - PyAggResourceMulti + PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti FEED_ATTRS = {'title': {'type': str}, 'description': {'type': str}, 'link': {'type': str}, 'user_id': {'type': int}, + 'category_id': {'type': int}, 'site_link': {'type': str}, 'enabled': {'type': bool, 'default': True}, 'etag': {'type': str, 'default': ''}, -- cgit