base modification in model for category support

10 files changed, 32 insertions, 28 deletions

diff --git a/src/crawler.py b/src/crawler.py
index 1a759945..0598c418 100644
--- a/src/crawler.py
+++ b/src/crawler.py
@@ -58,9 +58,6 @@ else:
 async def get(*args, **kwargs):
     #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
     try:
-        #logger.info("Fetching the feed: " + args[0])
-        #response = yield from aiohttp.request('GET', *args, **kwargs)
-        #return (yield from response.read_and_close(decode=False))
         data = feedparser.parse(args[0])
         return data
     except Exception as e:
@@ -131,7 +128,7 @@ async def insert_database(user, feed):
             new_articles.append(art_contr.create(**article))
             logger.info("New article % (%r) added.",
                         article['title'], article['link'])
-        except Exception as e:
+        except Exception:
             logger.exception("Error when inserting article in database:")
             continue
     return new_articles
diff --git a/src/manager.py b/src/manager.py
index e1f0878b..f7240670 100755
--- a/src/manager.py
+++ b/src/manager.py
@@ -12,6 +12,7 @@ Migrate(application, db)
 manager = Manager(application)
 manager.add_command('db', MigrateCommand)
 
+
 @manager.command
 def db_empty():
     "Will drop every datas stocked in db."
@@ -19,6 +20,7 @@ def db_empty():
         populate_g()
         web.models.db_empty(db)
 
+
 @manager.command
 def db_create():
     "Will create the database from conf parameters."
@@ -26,6 +28,7 @@ def db_create():
         populate_g()
         web.models.db_create(db)
 
+
 @manager.command
 def fetch(limit=100, retreive_all=False):
     "Crawl the feeds with the client crawler."
@@ -34,6 +37,7 @@ def fetch(limit=100, retreive_all=False):
     scheduler.run(limit=limit, retreive_all=retreive_all)
     scheduler.wait()
 
+
 @manager.command
 def fetch_asyncio(user_id, feed_id):
     "Crawl the feeds with asyncio."
@@ -63,7 +67,7 @@ def fetch_asyncio(user_id, feed_id):
             if user.activation_key == "":
                 print("Fetching articles for " + user.nickname)
                 g.user = user
-                feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
+                crawler.retrieve_feed(loop, g.user, feed_id)
         loop.close()
 
 from scripts.probes import ArticleProbe, FeedProbe
diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py
index 8b6926b7..a8788f46 100644
--- a/src/web/controllers/article.py
+++ b/src/web/controllers/article.py
@@ -34,8 +34,7 @@ class ArticleController(AbstractController):
                               .group_by(Article.feed_id).all())
 
     def count_by_user_id(self, **filters):
-        return dict(db.session.query(Article.user_id,
-                                            func.count(Article.id))
+        return dict(db.session.query(Article.user_id, func.count(Article.id))
                               .filter(*self._to_filters(**filters))
                               .group_by(Article.user_id).all())
 
diff --git a/src/web/lib/crawler.py b/src/web/lib/crawler.py
index 7343ea4d..f480fe96 100644
--- a/src/web/lib/crawler.py
+++ b/src/web/lib/crawler.py
@@ -18,7 +18,6 @@ import json
 import logging
 import feedparser
 from datetime import datetime, timedelta
-from functools import wraps
 from time import strftime, gmtime
 from concurrent.futures import ThreadPoolExecutor
 from requests_futures.sessions import FuturesSession
@@ -132,7 +131,7 @@ class PyAggUpdater(AbstractCrawler):
                 {key: "%s -> %s" % (up_feed[key], self.feed.get(key))
                  for key in up_feed if up_feed[key] != self.feed.get(key)})
 
-        future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
+        self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
 
 
 class FeedCrawler(AbstractCrawler):
@@ -144,8 +143,8 @@ class FeedCrawler(AbstractCrawler):
     def clean_feed(self):
         """Will reset the errors counters on a feed that have known errors"""
         if self.feed.get('error_count') or self.feed.get('last_error'):
-            future = self.query_pyagg('put', 'feed/%d' % self.feed['id'],
-                                      {'error_count': 0, 'last_error': ''})
+            self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+                             {'error_count': 0, 'last_error': ''})
 
     def callback(self, response):
         """will fetch the feed and interprete results (304, etag) or will
diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py
index 54168279..81ebe361 100644
--- a/src/web/models/__init__.py
+++ b/src/web/models/__init__.py
@@ -31,8 +31,9 @@ from .role import Role
 from .user import User
 from .article import Article
 from .icon import Icon
+from .category import Category
 
-__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon']
+__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category']
 
 import os
 
diff --git a/src/web/models/article.py b/src/web/models/article.py
index 54eefe75..44a4234d 100644
--- a/src/web/models/article.py
+++ b/src/web/models/article.py
@@ -32,10 +32,8 @@ from sqlalchemy import asc, desc
 
 
 class Article(db.Model):
-    """
-    Represent an article from a feed.
-    """
-    id = db.Column(db.Integer, primary_key=True)
+    "Represent an article from a feed."
+    id = db.Column(db.Integer(), primary_key=True)
     entry_id = db.Column(db.String())
     link = db.Column(db.String())
     title = db.Column(db.String())
@@ -46,8 +44,9 @@ class Article(db.Model):
     date = db.Column(db.DateTime(), default=datetime.now)
     retrieved_date = db.Column(db.DateTime(), default=datetime.now)
 
-    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
-    feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
+    user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+    feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id'))
+    category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
 
     def previous_article(self):
         """
diff --git a/src/web/models/feed.py b/src/web/models/feed.py
index 59456a7f..87616e02 100644
--- a/src/web/models/feed.py
+++ b/src/web/models/feed.py
@@ -56,6 +56,7 @@ class Feed(db.Model):
     # relationship
     icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
     user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+    category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
     articles = db.relationship('Article', backref='source', lazy='dynamic',
                                cascade='all,delete-orphan',
                                order_by=desc("Article.date"))
diff --git a/src/web/views/api/__init__.py b/src/web/views/api/__init__.py
index 24472ebe..ed3bd702 100644
--- a/src/web/views/api/__init__.py
+++ b/src/web/views/api/__init__.py
@@ -26,6 +26,6 @@ __revision__ = "$Date: 2014/07/05 $"
 __copyright__ = "Copyright (c) Cedric Bonhomme"
 __license__ = "AGPLv3"
 
-from web.views.api import article, feed
+from web.views.api import article, feed, category
 
-__all__ = ['article', 'feed']
+__all__ = ['article', 'feed', 'category']
diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py
index 51844b20..23c5c495 100644
--- a/src/web/views/api/article.py
+++ b/src/web/views/api/article.py
@@ -13,12 +13,15 @@ from web.views.api.common import PyAggAbstractResource,\
 
 ARTICLE_ATTRS = {'user_id': {'type': int},
                  'feed_id': {'type': int},
+                 'category_id': {'type': int},
                  'entry_id': {'type': str},
                  'link': {'type': str},
                  'title': {'type': str},
-                 'readed': {'type': bool}, 'like': {'type': bool},
+                 'readed': {'type': bool},
+                 'like': {'type': bool},
                  'content': {'type': str},
-                 'date': {'type': str}, 'retrieved_date': {'type': str}}
+                 'date': {'type': str},
+                 'retrieved_date': {'type': str}}
 
 
 class ArticleNewAPI(PyAggResourceNew):
diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py
index 2bb9814f..dd9919bf 100644
--- a/src/web/views/api/feed.py
+++ b/src/web/views/api/feed.py
@@ -4,19 +4,20 @@
 from flask import g
 
 from web.controllers.feed import (FeedController,
-                                           DEFAULT_MAX_ERROR,
-                                           DEFAULT_LIMIT,
-                                           DEFAULT_REFRESH_RATE)
+                                  DEFAULT_MAX_ERROR,
+                                  DEFAULT_LIMIT,
+                                  DEFAULT_REFRESH_RATE)
 
 from web.views.api.common import PyAggAbstractResource, \
-                                          PyAggResourceNew, \
-                                          PyAggResourceExisting, \
-                                          PyAggResourceMulti
+                                 PyAggResourceNew, \
+                                 PyAggResourceExisting, \
+                                 PyAggResourceMulti
 
 FEED_ATTRS = {'title': {'type': str},
               'description': {'type': str},
               'link': {'type': str},
               'user_id': {'type': int},
+              'category_id': {'type': int},
               'site_link': {'type': str},
               'enabled': {'type': bool, 'default': True},
               'etag': {'type': str, 'default': ''},