diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/crawler.py | 5 | ||||
| -rwxr-xr-x | src/manager.py | 6 | ||||
| -rw-r--r-- | src/web/controllers/article.py | 3 | ||||
| -rw-r--r-- | src/web/lib/crawler.py | 7 | ||||
| -rw-r--r-- | src/web/models/__init__.py | 3 | ||||
| -rw-r--r-- | src/web/models/article.py | 11 | ||||
| -rw-r--r-- | src/web/models/feed.py | 1 | ||||
| -rw-r--r-- | src/web/views/api/__init__.py | 4 | ||||
| -rw-r--r-- | src/web/views/api/article.py | 7 | ||||
| -rw-r--r-- | src/web/views/api/feed.py | 13 |
10 files changed, 32 insertions, 28 deletions
diff --git a/src/crawler.py b/src/crawler.py index 1a759945..0598c418 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -58,9 +58,6 @@ else: async def get(*args, **kwargs): #kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False) try: - #logger.info("Fetching the feed: " + args[0]) - #response = yield from aiohttp.request('GET', *args, **kwargs) - #return (yield from response.read_and_close(decode=False)) data = feedparser.parse(args[0]) return data except Exception as e: @@ -131,7 +128,7 @@ async def insert_database(user, feed): new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) - except Exception as e: + except Exception: logger.exception("Error when inserting article in database:") continue return new_articles diff --git a/src/manager.py b/src/manager.py index e1f0878b..f7240670 100755 --- a/src/manager.py +++ b/src/manager.py @@ -12,6 +12,7 @@ Migrate(application, db) manager = Manager(application) manager.add_command('db', MigrateCommand) + @manager.command def db_empty(): "Will drop every datas stocked in db." @@ -19,6 +20,7 @@ def db_empty(): populate_g() web.models.db_empty(db) + @manager.command def db_create(): "Will create the database from conf parameters." @@ -26,6 +28,7 @@ def db_create(): populate_g() web.models.db_create(db) + @manager.command def fetch(limit=100, retreive_all=False): "Crawl the feeds with the client crawler." @@ -34,6 +37,7 @@ def fetch(limit=100, retreive_all=False): scheduler.run(limit=limit, retreive_all=retreive_all) scheduler.wait() + @manager.command def fetch_asyncio(user_id, feed_id): "Crawl the feeds with asyncio." @@ -63,7 +67,7 @@ def fetch_asyncio(user_id, feed_id): if user.activation_key == "": print("Fetching articles for " + user.nickname) g.user = user - feed_getter = crawler.retrieve_feed(loop, g.user, feed_id) + crawler.retrieve_feed(loop, g.user, feed_id) loop.close() from scripts.probes import ArticleProbe, FeedProbe diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index 8b6926b7..a8788f46 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -34,8 +34,7 @@ class ArticleController(AbstractController): .group_by(Article.feed_id).all()) def count_by_user_id(self, **filters): - return dict(db.session.query(Article.user_id, - func.count(Article.id)) + return dict(db.session.query(Article.user_id, func.count(Article.id)) .filter(*self._to_filters(**filters)) .group_by(Article.user_id).all()) diff --git a/src/web/lib/crawler.py b/src/web/lib/crawler.py index 7343ea4d..f480fe96 100644 --- a/src/web/lib/crawler.py +++ b/src/web/lib/crawler.py @@ -18,7 +18,6 @@ import json import logging import feedparser from datetime import datetime, timedelta -from functools import wraps from time import strftime, gmtime from concurrent.futures import ThreadPoolExecutor from requests_futures.sessions import FuturesSession @@ -132,7 +131,7 @@ class PyAggUpdater(AbstractCrawler): {key: "%s -> %s" % (up_feed[key], self.feed.get(key)) for key in up_feed if up_feed[key] != self.feed.get(key)}) - future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed) class FeedCrawler(AbstractCrawler): @@ -144,8 +143,8 @@ class FeedCrawler(AbstractCrawler): def clean_feed(self): """Will reset the errors counters on a feed that have known errors""" if self.feed.get('error_count') or self.feed.get('last_error'): - future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], - {'error_count': 0, 'last_error': ''}) + self.query_pyagg('put', 'feed/%d' % self.feed['id'], + {'error_count': 0, 'last_error': ''}) def callback(self, response): """will fetch the feed and interprete results (304, etag) or will diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py index 54168279..81ebe361 100644 --- a/src/web/models/__init__.py +++ b/src/web/models/__init__.py @@ -31,8 +31,9 @@ from .role import Role from .user import User from .article import Article from .icon import Icon +from .category import Category -__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon'] +__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category'] import os diff --git a/src/web/models/article.py b/src/web/models/article.py index 54eefe75..44a4234d 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -32,10 +32,8 @@ from sqlalchemy import asc, desc class Article(db.Model): - """ - Represent an article from a feed. - """ - id = db.Column(db.Integer, primary_key=True) + "Represent an article from a feed." + id = db.Column(db.Integer(), primary_key=True) entry_id = db.Column(db.String()) link = db.Column(db.String()) title = db.Column(db.String()) @@ -46,8 +44,9 @@ class Article(db.Model): date = db.Column(db.DateTime(), default=datetime.now) retrieved_date = db.Column(db.DateTime(), default=datetime.now) - user_id = db.Column(db.Integer, db.ForeignKey('user.id')) - feed_id = db.Column(db.Integer, db.ForeignKey('feed.id')) + user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) + feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id')) + category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) def previous_article(self): """ diff --git a/src/web/models/feed.py b/src/web/models/feed.py index 59456a7f..87616e02 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -56,6 +56,7 @@ class Feed(db.Model): # relationship icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None) user_id = db.Column(db.Integer(), db.ForeignKey('user.id')) + category_id = db.Column(db.Integer(), db.ForeignKey('category.id')) articles = db.relationship('Article', backref='source', lazy='dynamic', cascade='all,delete-orphan', order_by=desc("Article.date")) diff --git a/src/web/views/api/__init__.py b/src/web/views/api/__init__.py index 24472ebe..ed3bd702 100644 --- a/src/web/views/api/__init__.py +++ b/src/web/views/api/__init__.py @@ -26,6 +26,6 @@ __revision__ = "$Date: 2014/07/05 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "AGPLv3" -from web.views.api import article, feed +from web.views.api import article, feed, category -__all__ = ['article', 'feed'] +__all__ = ['article', 'feed', 'category'] diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py index 51844b20..23c5c495 100644 --- a/src/web/views/api/article.py +++ b/src/web/views/api/article.py @@ -13,12 +13,15 @@ from web.views.api.common import PyAggAbstractResource,\ ARTICLE_ATTRS = {'user_id': {'type': int}, 'feed_id': {'type': int}, + 'category_id': {'type': int}, 'entry_id': {'type': str}, 'link': {'type': str}, 'title': {'type': str}, - 'readed': {'type': bool}, 'like': {'type': bool}, + 'readed': {'type': bool}, + 'like': {'type': bool}, 'content': {'type': str}, - 'date': {'type': str}, 'retrieved_date': {'type': str}} + 'date': {'type': str}, + 'retrieved_date': {'type': str}} class ArticleNewAPI(PyAggResourceNew): diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py index 2bb9814f..dd9919bf 100644 --- a/src/web/views/api/feed.py +++ b/src/web/views/api/feed.py @@ -4,19 +4,20 @@ from flask import g from web.controllers.feed import (FeedController, - DEFAULT_MAX_ERROR, - DEFAULT_LIMIT, - DEFAULT_REFRESH_RATE) + DEFAULT_MAX_ERROR, + DEFAULT_LIMIT, + DEFAULT_REFRESH_RATE) from web.views.api.common import PyAggAbstractResource, \ - PyAggResourceNew, \ - PyAggResourceExisting, \ - PyAggResourceMulti + PyAggResourceNew, \ + PyAggResourceExisting, \ + PyAggResourceMulti FEED_ATTRS = {'title': {'type': str}, 'description': {'type': str}, 'link': {'type': str}, 'user_id': {'type': int}, + 'category_id': {'type': int}, 'site_link': {'type': str}, 'enabled': {'type': bool, 'default': True}, 'etag': {'type': str, 'default': ''}, |