aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--migrations/versions/3f83bfe93fc_adding_category.py42
-rw-r--r--pyaggr3g470r/controllers/category.py6
-rw-r--r--pyaggr3g470r/models/category.py8
-rw-r--r--pyaggr3g470r/views/api/category.py31
-rw-r--r--src/crawler.py5
-rwxr-xr-xsrc/manager.py6
-rw-r--r--src/web/controllers/article.py3
-rw-r--r--src/web/lib/crawler.py7
-rw-r--r--src/web/models/__init__.py3
-rw-r--r--src/web/models/article.py11
-rw-r--r--src/web/models/feed.py1
-rw-r--r--src/web/views/api/__init__.py4
-rw-r--r--src/web/views/api/article.py7
-rw-r--r--src/web/views/api/feed.py13
14 files changed, 119 insertions, 28 deletions
diff --git a/migrations/versions/3f83bfe93fc_adding_category.py b/migrations/versions/3f83bfe93fc_adding_category.py
new file mode 100644
index 00000000..bd65a01f
--- /dev/null
+++ b/migrations/versions/3f83bfe93fc_adding_category.py
@@ -0,0 +1,42 @@
+"""adding category
+
+Revision ID: 3f83bfe93fc
+Revises: 25ca960a207
+Create Date: 2015-09-01 14:15:04.212563
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3f83bfe93fc'
+down_revision = '25ca960a207'
+
+import conf
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.create_table('category',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+ sa.PrimaryKeyConstraint('id'))
+ op.add_column('article',
+ sa.Column('category_id', sa.Integer(), nullable=True))
+ op.add_column('feed',
+ sa.Column('category_id', sa.Integer(), nullable=True))
+ if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI:
+ op.create_foreign_key(None, 'article', 'category',
+ ['category_id'], ['id'])
+ op.create_foreign_key(None, 'feed', 'category',
+ ['category_id'], ['id'])
+
+
+def downgrade():
+ if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI:
+ op.drop_constraint(None, 'feed', type_='foreignkey')
+ op.drop_constraint(None, 'feed', type_='foreignkey')
+ op.drop_column('feed', 'category_id')
+ op.drop_constraint(None, 'article', type_='foreignkey')
+ op.drop_column('article', 'category_id')
+ op.drop_table('category')
diff --git a/pyaggr3g470r/controllers/category.py b/pyaggr3g470r/controllers/category.py
new file mode 100644
index 00000000..b6fc591c
--- /dev/null
+++ b/pyaggr3g470r/controllers/category.py
@@ -0,0 +1,6 @@
+from .abstract import AbstractController
+from pyaggr3g470r.models import Category
+
+
+class CategoryController(AbstractController):
+ _db_cls = Category
diff --git a/pyaggr3g470r/models/category.py b/pyaggr3g470r/models/category.py
new file mode 100644
index 00000000..513227a5
--- /dev/null
+++ b/pyaggr3g470r/models/category.py
@@ -0,0 +1,8 @@
+from bootstrap import db
+
+
+class Category(db.Model):
+ id = db.Column(db.Integer(), primary_key=True)
+ name = db.Column(db.String())
+
+ user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
diff --git a/pyaggr3g470r/views/api/category.py b/pyaggr3g470r/views/api/category.py
new file mode 100644
index 00000000..31f46751
--- /dev/null
+++ b/pyaggr3g470r/views/api/category.py
@@ -0,0 +1,31 @@
+from flask import g
+
+from pyaggr3g470r.controllers.category import CategoryController
+from pyaggr3g470r.views.api.common import (PyAggResourceNew,
+ PyAggResourceExisting,
+ PyAggResourceMulti)
+
+
+CAT_ATTRS = {'name': {'type': str},
+ 'user_id': {'type': int}}
+
+
+class CategoryNewAPI(PyAggResourceNew):
+ controller_cls = CategoryController
+ attrs = CAT_ATTRS
+
+
+class CategoryAPI(PyAggResourceExisting):
+ controller_cls = CategoryController
+ attrs = CAT_ATTRS
+
+
+class CategoriesAPI(PyAggResourceMulti):
+ controller_cls = CategoryController
+ attrs = CAT_ATTRS
+
+
+g.api.add_resource(CategoryNewAPI, '/category', endpoint='category_new.json')
+g.api.add_resource(CategoryAPI, '/category/<int:obj_id>',
+ endpoint='category.json')
+g.api.add_resource(CategoriesAPI, '/categories', endpoint='categories.json')
diff --git a/src/crawler.py b/src/crawler.py
index 1a759945..0598c418 100644
--- a/src/crawler.py
+++ b/src/crawler.py
@@ -58,9 +58,6 @@ else:
async def get(*args, **kwargs):
#kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
try:
- #logger.info("Fetching the feed: " + args[0])
- #response = yield from aiohttp.request('GET', *args, **kwargs)
- #return (yield from response.read_and_close(decode=False))
data = feedparser.parse(args[0])
return data
except Exception as e:
@@ -131,7 +128,7 @@ async def insert_database(user, feed):
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
article['title'], article['link'])
- except Exception as e:
+ except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
diff --git a/src/manager.py b/src/manager.py
index e1f0878b..f7240670 100755
--- a/src/manager.py
+++ b/src/manager.py
@@ -12,6 +12,7 @@ Migrate(application, db)
manager = Manager(application)
manager.add_command('db', MigrateCommand)
+
@manager.command
def db_empty():
"Will drop every datas stocked in db."
@@ -19,6 +20,7 @@ def db_empty():
populate_g()
web.models.db_empty(db)
+
@manager.command
def db_create():
"Will create the database from conf parameters."
@@ -26,6 +28,7 @@ def db_create():
populate_g()
web.models.db_create(db)
+
@manager.command
def fetch(limit=100, retreive_all=False):
"Crawl the feeds with the client crawler."
@@ -34,6 +37,7 @@ def fetch(limit=100, retreive_all=False):
scheduler.run(limit=limit, retreive_all=retreive_all)
scheduler.wait()
+
@manager.command
def fetch_asyncio(user_id, feed_id):
"Crawl the feeds with asyncio."
@@ -63,7 +67,7 @@ def fetch_asyncio(user_id, feed_id):
if user.activation_key == "":
print("Fetching articles for " + user.nickname)
g.user = user
- feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
+ crawler.retrieve_feed(loop, g.user, feed_id)
loop.close()
from scripts.probes import ArticleProbe, FeedProbe
diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py
index 8b6926b7..a8788f46 100644
--- a/src/web/controllers/article.py
+++ b/src/web/controllers/article.py
@@ -34,8 +34,7 @@ class ArticleController(AbstractController):
.group_by(Article.feed_id).all())
def count_by_user_id(self, **filters):
- return dict(db.session.query(Article.user_id,
- func.count(Article.id))
+ return dict(db.session.query(Article.user_id, func.count(Article.id))
.filter(*self._to_filters(**filters))
.group_by(Article.user_id).all())
diff --git a/src/web/lib/crawler.py b/src/web/lib/crawler.py
index 7343ea4d..f480fe96 100644
--- a/src/web/lib/crawler.py
+++ b/src/web/lib/crawler.py
@@ -18,7 +18,6 @@ import json
import logging
import feedparser
from datetime import datetime, timedelta
-from functools import wraps
from time import strftime, gmtime
from concurrent.futures import ThreadPoolExecutor
from requests_futures.sessions import FuturesSession
@@ -132,7 +131,7 @@ class PyAggUpdater(AbstractCrawler):
{key: "%s -> %s" % (up_feed[key], self.feed.get(key))
for key in up_feed if up_feed[key] != self.feed.get(key)})
- future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
class FeedCrawler(AbstractCrawler):
@@ -144,8 +143,8 @@ class FeedCrawler(AbstractCrawler):
def clean_feed(self):
"""Will reset the errors counters on a feed that have known errors"""
if self.feed.get('error_count') or self.feed.get('last_error'):
- future = self.query_pyagg('put', 'feed/%d' % self.feed['id'],
- {'error_count': 0, 'last_error': ''})
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+ {'error_count': 0, 'last_error': ''})
def callback(self, response):
"""will fetch the feed and interprete results (304, etag) or will
diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py
index 54168279..81ebe361 100644
--- a/src/web/models/__init__.py
+++ b/src/web/models/__init__.py
@@ -31,8 +31,9 @@ from .role import Role
from .user import User
from .article import Article
from .icon import Icon
+from .category import Category
-__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon']
+__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category']
import os
diff --git a/src/web/models/article.py b/src/web/models/article.py
index 54eefe75..44a4234d 100644
--- a/src/web/models/article.py
+++ b/src/web/models/article.py
@@ -32,10 +32,8 @@ from sqlalchemy import asc, desc
class Article(db.Model):
- """
- Represent an article from a feed.
- """
- id = db.Column(db.Integer, primary_key=True)
+ "Represent an article from a feed."
+ id = db.Column(db.Integer(), primary_key=True)
entry_id = db.Column(db.String())
link = db.Column(db.String())
title = db.Column(db.String())
@@ -46,8 +44,9 @@ class Article(db.Model):
date = db.Column(db.DateTime(), default=datetime.now)
retrieved_date = db.Column(db.DateTime(), default=datetime.now)
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
- feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
+ user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+ feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id'))
+ category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
def previous_article(self):
"""
diff --git a/src/web/models/feed.py b/src/web/models/feed.py
index 59456a7f..87616e02 100644
--- a/src/web/models/feed.py
+++ b/src/web/models/feed.py
@@ -56,6 +56,7 @@ class Feed(db.Model):
# relationship
icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+ category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
articles = db.relationship('Article', backref='source', lazy='dynamic',
cascade='all,delete-orphan',
order_by=desc("Article.date"))
diff --git a/src/web/views/api/__init__.py b/src/web/views/api/__init__.py
index 24472ebe..ed3bd702 100644
--- a/src/web/views/api/__init__.py
+++ b/src/web/views/api/__init__.py
@@ -26,6 +26,6 @@ __revision__ = "$Date: 2014/07/05 $"
__copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "AGPLv3"
-from web.views.api import article, feed
+from web.views.api import article, feed, category
-__all__ = ['article', 'feed']
+__all__ = ['article', 'feed', 'category']
diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py
index 51844b20..23c5c495 100644
--- a/src/web/views/api/article.py
+++ b/src/web/views/api/article.py
@@ -13,12 +13,15 @@ from web.views.api.common import PyAggAbstractResource,\
ARTICLE_ATTRS = {'user_id': {'type': int},
'feed_id': {'type': int},
+ 'category_id': {'type': int},
'entry_id': {'type': str},
'link': {'type': str},
'title': {'type': str},
- 'readed': {'type': bool}, 'like': {'type': bool},
+ 'readed': {'type': bool},
+ 'like': {'type': bool},
'content': {'type': str},
- 'date': {'type': str}, 'retrieved_date': {'type': str}}
+ 'date': {'type': str},
+ 'retrieved_date': {'type': str}}
class ArticleNewAPI(PyAggResourceNew):
diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py
index 2bb9814f..dd9919bf 100644
--- a/src/web/views/api/feed.py
+++ b/src/web/views/api/feed.py
@@ -4,19 +4,20 @@
from flask import g
from web.controllers.feed import (FeedController,
- DEFAULT_MAX_ERROR,
- DEFAULT_LIMIT,
- DEFAULT_REFRESH_RATE)
+ DEFAULT_MAX_ERROR,
+ DEFAULT_LIMIT,
+ DEFAULT_REFRESH_RATE)
from web.views.api.common import PyAggAbstractResource, \
- PyAggResourceNew, \
- PyAggResourceExisting, \
- PyAggResourceMulti
+ PyAggResourceNew, \
+ PyAggResourceExisting, \
+ PyAggResourceMulti
FEED_ATTRS = {'title': {'type': str},
'description': {'type': str},
'link': {'type': str},
'user_id': {'type': int},
+ 'category_id': {'type': int},
'site_link': {'type': str},
'enabled': {'type': bool, 'default': True},
'etag': {'type': str, 'default': ''},
bgstack15