aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorFrançois Schmidts <francois.schmidts@gmail.com>2015-09-11 18:28:12 +0200
committerFrançois Schmidts <francois.schmidts@gmail.com>2016-01-26 23:46:30 +0100
commit462f6d3b21558ed0a283c24e0e0332eac6ccbbb3 (patch)
tree451c583b5f47bbf6e38743881c66f2f27371bd82 /src
parentmoving the root of source code from / to /src/ (diff)
downloadnewspipe-462f6d3b21558ed0a283c24e0e0332eac6ccbbb3.tar.gz
newspipe-462f6d3b21558ed0a283c24e0e0332eac6ccbbb3.tar.bz2
newspipe-462f6d3b21558ed0a283c24e0e0332eac6ccbbb3.zip
base modification in model for category support
Diffstat (limited to 'src')
-rw-r--r--src/crawler.py5
-rwxr-xr-xsrc/manager.py6
-rw-r--r--src/web/controllers/article.py3
-rw-r--r--src/web/lib/crawler.py7
-rw-r--r--src/web/models/__init__.py3
-rw-r--r--src/web/models/article.py11
-rw-r--r--src/web/models/feed.py1
-rw-r--r--src/web/views/api/__init__.py4
-rw-r--r--src/web/views/api/article.py7
-rw-r--r--src/web/views/api/feed.py13
10 files changed, 32 insertions, 28 deletions
diff --git a/src/crawler.py b/src/crawler.py
index 1a759945..0598c418 100644
--- a/src/crawler.py
+++ b/src/crawler.py
@@ -58,9 +58,6 @@ else:
async def get(*args, **kwargs):
#kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False)
try:
- #logger.info("Fetching the feed: " + args[0])
- #response = yield from aiohttp.request('GET', *args, **kwargs)
- #return (yield from response.read_and_close(decode=False))
data = feedparser.parse(args[0])
return data
except Exception as e:
@@ -131,7 +128,7 @@ async def insert_database(user, feed):
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
article['title'], article['link'])
- except Exception as e:
+ except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
diff --git a/src/manager.py b/src/manager.py
index e1f0878b..f7240670 100755
--- a/src/manager.py
+++ b/src/manager.py
@@ -12,6 +12,7 @@ Migrate(application, db)
manager = Manager(application)
manager.add_command('db', MigrateCommand)
+
@manager.command
def db_empty():
"Will drop every datas stocked in db."
@@ -19,6 +20,7 @@ def db_empty():
populate_g()
web.models.db_empty(db)
+
@manager.command
def db_create():
"Will create the database from conf parameters."
@@ -26,6 +28,7 @@ def db_create():
populate_g()
web.models.db_create(db)
+
@manager.command
def fetch(limit=100, retreive_all=False):
"Crawl the feeds with the client crawler."
@@ -34,6 +37,7 @@ def fetch(limit=100, retreive_all=False):
scheduler.run(limit=limit, retreive_all=retreive_all)
scheduler.wait()
+
@manager.command
def fetch_asyncio(user_id, feed_id):
"Crawl the feeds with asyncio."
@@ -63,7 +67,7 @@ def fetch_asyncio(user_id, feed_id):
if user.activation_key == "":
print("Fetching articles for " + user.nickname)
g.user = user
- feed_getter = crawler.retrieve_feed(loop, g.user, feed_id)
+ crawler.retrieve_feed(loop, g.user, feed_id)
loop.close()
from scripts.probes import ArticleProbe, FeedProbe
diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py
index 8b6926b7..a8788f46 100644
--- a/src/web/controllers/article.py
+++ b/src/web/controllers/article.py
@@ -34,8 +34,7 @@ class ArticleController(AbstractController):
.group_by(Article.feed_id).all())
def count_by_user_id(self, **filters):
- return dict(db.session.query(Article.user_id,
- func.count(Article.id))
+ return dict(db.session.query(Article.user_id, func.count(Article.id))
.filter(*self._to_filters(**filters))
.group_by(Article.user_id).all())
diff --git a/src/web/lib/crawler.py b/src/web/lib/crawler.py
index 7343ea4d..f480fe96 100644
--- a/src/web/lib/crawler.py
+++ b/src/web/lib/crawler.py
@@ -18,7 +18,6 @@ import json
import logging
import feedparser
from datetime import datetime, timedelta
-from functools import wraps
from time import strftime, gmtime
from concurrent.futures import ThreadPoolExecutor
from requests_futures.sessions import FuturesSession
@@ -132,7 +131,7 @@ class PyAggUpdater(AbstractCrawler):
{key: "%s -> %s" % (up_feed[key], self.feed.get(key))
for key in up_feed if up_feed[key] != self.feed.get(key)})
- future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
class FeedCrawler(AbstractCrawler):
@@ -144,8 +143,8 @@ class FeedCrawler(AbstractCrawler):
def clean_feed(self):
"""Will reset the errors counters on a feed that have known errors"""
if self.feed.get('error_count') or self.feed.get('last_error'):
- future = self.query_pyagg('put', 'feed/%d' % self.feed['id'],
- {'error_count': 0, 'last_error': ''})
+ self.query_pyagg('put', 'feed/%d' % self.feed['id'],
+ {'error_count': 0, 'last_error': ''})
def callback(self, response):
"""will fetch the feed and interprete results (304, etag) or will
diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py
index 54168279..81ebe361 100644
--- a/src/web/models/__init__.py
+++ b/src/web/models/__init__.py
@@ -31,8 +31,9 @@ from .role import Role
from .user import User
from .article import Article
from .icon import Icon
+from .category import Category
-__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon']
+__all__ = ['Feed', 'Role', 'User', 'Article', 'Icon', 'Category']
import os
diff --git a/src/web/models/article.py b/src/web/models/article.py
index 54eefe75..44a4234d 100644
--- a/src/web/models/article.py
+++ b/src/web/models/article.py
@@ -32,10 +32,8 @@ from sqlalchemy import asc, desc
class Article(db.Model):
- """
- Represent an article from a feed.
- """
- id = db.Column(db.Integer, primary_key=True)
+ "Represent an article from a feed."
+ id = db.Column(db.Integer(), primary_key=True)
entry_id = db.Column(db.String())
link = db.Column(db.String())
title = db.Column(db.String())
@@ -46,8 +44,9 @@ class Article(db.Model):
date = db.Column(db.DateTime(), default=datetime.now)
retrieved_date = db.Column(db.DateTime(), default=datetime.now)
- user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
- feed_id = db.Column(db.Integer, db.ForeignKey('feed.id'))
+ user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+ feed_id = db.Column(db.Integer(), db.ForeignKey('feed.id'))
+ category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
def previous_article(self):
"""
diff --git a/src/web/models/feed.py b/src/web/models/feed.py
index 59456a7f..87616e02 100644
--- a/src/web/models/feed.py
+++ b/src/web/models/feed.py
@@ -56,6 +56,7 @@ class Feed(db.Model):
# relationship
icon_url = db.Column(db.String(), db.ForeignKey('icon.url'), default=None)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
+ category_id = db.Column(db.Integer(), db.ForeignKey('category.id'))
articles = db.relationship('Article', backref='source', lazy='dynamic',
cascade='all,delete-orphan',
order_by=desc("Article.date"))
diff --git a/src/web/views/api/__init__.py b/src/web/views/api/__init__.py
index 24472ebe..ed3bd702 100644
--- a/src/web/views/api/__init__.py
+++ b/src/web/views/api/__init__.py
@@ -26,6 +26,6 @@ __revision__ = "$Date: 2014/07/05 $"
__copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "AGPLv3"
-from web.views.api import article, feed
+from web.views.api import article, feed, category
-__all__ = ['article', 'feed']
+__all__ = ['article', 'feed', 'category']
diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py
index 51844b20..23c5c495 100644
--- a/src/web/views/api/article.py
+++ b/src/web/views/api/article.py
@@ -13,12 +13,15 @@ from web.views.api.common import PyAggAbstractResource,\
ARTICLE_ATTRS = {'user_id': {'type': int},
'feed_id': {'type': int},
+ 'category_id': {'type': int},
'entry_id': {'type': str},
'link': {'type': str},
'title': {'type': str},
- 'readed': {'type': bool}, 'like': {'type': bool},
+ 'readed': {'type': bool},
+ 'like': {'type': bool},
'content': {'type': str},
- 'date': {'type': str}, 'retrieved_date': {'type': str}}
+ 'date': {'type': str},
+ 'retrieved_date': {'type': str}}
class ArticleNewAPI(PyAggResourceNew):
diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py
index 2bb9814f..dd9919bf 100644
--- a/src/web/views/api/feed.py
+++ b/src/web/views/api/feed.py
@@ -4,19 +4,20 @@
from flask import g
from web.controllers.feed import (FeedController,
- DEFAULT_MAX_ERROR,
- DEFAULT_LIMIT,
- DEFAULT_REFRESH_RATE)
+ DEFAULT_MAX_ERROR,
+ DEFAULT_LIMIT,
+ DEFAULT_REFRESH_RATE)
from web.views.api.common import PyAggAbstractResource, \
- PyAggResourceNew, \
- PyAggResourceExisting, \
- PyAggResourceMulti
+ PyAggResourceNew, \
+ PyAggResourceExisting, \
+ PyAggResourceMulti
FEED_ATTRS = {'title': {'type': str},
'description': {'type': str},
'link': {'type': str},
'user_id': {'type': int},
+ 'category_id': {'type': int},
'site_link': {'type': str},
'enabled': {'type': bool, 'default': True},
'etag': {'type': str, 'default': ''},
bgstack15