From 83b4671112914081b3912dc06f006ed84147386a Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 4 Aug 2015 10:33:15 +0200 Subject: more elegant method to check for icons --- pyaggr3g470r/controllers/feed.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyaggr3g470r/controllers/feed.py b/pyaggr3g470r/controllers/feed.py index 6b3c4fb5..a8e96217 100644 --- a/pyaggr3g470r/controllers/feed.py +++ b/pyaggr3g470r/controllers/feed.py @@ -21,7 +21,6 @@ import logging from datetime import datetime, timedelta -from werkzeug.exceptions import NotFound import conf from .abstract import AbstractController @@ -59,9 +58,7 @@ class FeedController(AbstractController): if not attrs.get('icon_url'): return icon_contr = IconController() - try: - icon_contr.get(url=attrs['icon_url']) - except NotFound: + if not icon_contr.read(url=attrs['icon_url']).count(): icon_contr.create(**{'url': attrs['icon_url']}) def create(self, **attrs): -- cgit From dc96d2326411f45a0e1a14bdec2265821cceb65e Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 4 Aug 2015 10:49:42 +0200 Subject: making feeds/fetchable and articles/challenge returns 204 on empty --- pyaggr3g470r/lib/crawler.py | 29 +++++++++++++++++------------ pyaggr3g470r/views/api/article.py | 3 ++- pyaggr3g470r/views/api/feed.py | 3 ++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index 216e7a96..cca3245e 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -116,19 +116,21 @@ class PyAggUpdater(AbstractCrawler): """Will process the result from the challenge, creating missing article and updating the feed""" AbstractCrawler.__counter__ -= 1 - results = response.result().json() - logger.debug('%r %r - %d entries were not matched and will be created', - self.feed['id'], self.feed['title'], len(results)) article_created = False - for id_to_create in results: - article_created = True - entry = construct_article( - self.entries[tuple(sorted(id_to_create.items()))], - self.feed) - logger.info('%r %r - creating %r for %r - %r', self.feed['id'], - self.feed['title'], entry['title'], entry['user_id'], - id_to_create) - self.query_pyagg('post', 'article', entry) + if response.result().status_code != 204: + results = response.result().json() + logger.debug('%r %r - %d entries were not matched ' + 'and will be created', + self.feed['id'], self.feed['title'], len(results)) + for id_to_create in results: + article_created = True + entry = construct_article( + self.entries[tuple(sorted(id_to_create.items()))], + self.feed) + logger.info('%r %r - creating %r for %r - %r', self.feed['id'], + self.feed['title'], entry['title'], + entry['user_id'], id_to_create) + self.query_pyagg('post', 'article', entry) logger.debug('%r %r - updating feed etag %r last_mod %r', self.feed['id'], self.feed['title'], @@ -263,6 +265,9 @@ class CrawlerScheduler(AbstractCrawler): AbstractCrawler.__counter__ -= 1 response = response.result() response.raise_for_status() + if response.status_code == 204: + logger.debug("No feed to fetch") + return feeds = response.json() logger.debug('%d to fetch %r', len(feeds), feeds) for feed in feeds: diff --git a/pyaggr3g470r/views/api/article.py b/pyaggr3g470r/views/api/article.py index 03ecdb18..d2969cb0 100644 --- a/pyaggr3g470r/views/api/article.py +++ b/pyaggr3g470r/views/api/article.py @@ -51,7 +51,8 @@ class ArticlesChallenge(PyAggAbstractResource): if key in id_dict: id_dict[key] = dateutil.parser.parse(id_dict[key]) - return self.wider_controller.challenge(parsed_args['ids']) + result = list(self.wider_controller.challenge(parsed_args['ids'])) + return result or None, 200 if result else 204 g.api.add_resource(ArticleNewAPI, '/article', endpoint='article_new.json') diff --git a/pyaggr3g470r/views/api/feed.py b/pyaggr3g470r/views/api/feed.py index 7d8cdf38..c80e9a9b 100644 --- a/pyaggr3g470r/views/api/feed.py +++ b/pyaggr3g470r/views/api/feed.py @@ -58,7 +58,8 @@ class FetchableFeedAPI(PyAggAbstractResource): contr = self.wider_controller else: contr = self.controller - return [feed for feed in contr.list_fetchable(**args)] + result = [feed for feed in contr.list_fetchable(**args)] + return result or None, 200 if result else 204 g.api.add_resource(FeedNewAPI, '/feed', endpoint='feed_new.json') -- cgit From e624d47ec2f3fad98f4312769408b36b6d2213d7 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 4 Aug 2015 14:45:42 +0200 Subject: redoing the way the crawler wait on itself --- pyaggr3g470r/lib/crawler.py | 62 +++++++++++---------------------------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index cca3245e..b599e2d4 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -33,10 +33,8 @@ API_ROOT = "api/v2.0/" class AbstractCrawler: __session__ = None - __counter__ = 0 def __init__(self, auth): - AbstractCrawler.__counter__ += 1 self.auth = auth self.session = self.get_session() self.url = conf.PLATFORM_URL @@ -50,30 +48,6 @@ class AbstractCrawler: cls.__session__.verify = False return cls.__session__ - @classmethod - def count_on_me(cls, func): - """A basic decorator which will count +1 at the begining of a call - and -1 at the end. It kinda allows us to wait for the __counter__ value - to be 0, meaning nothing is done anymore.""" - @wraps(func) - def wrapper(*args, **kwargs): - cls.__counter__ += 1 - try: - return func(*args, **kwargs) - except: - logger.exception('an error occured while %r', func) - finally: - cls.__counter__ -= 1 - return wrapper - - @classmethod - def get_counter_callback(cls): - cls.__counter__ += 1 - - def debump(*args, **kwargs): - cls.__counter__ -= 1 - return debump - def query_pyagg(self, method, urn, data=None): """A wrapper for internal call, method should be ones you can find on requests (header, post, get, options, ...), urn the distant @@ -89,17 +63,23 @@ class AbstractCrawler: 'User-Agent': 'pyaggr3g470r'}) @classmethod - def wait(cls, max_wait=600): + def wait(cls, max_wait=300, checks=5, wait_for=2): "See count_on_me, that method will just wait for the counter to be 0" - time.sleep(1) - second_waited = 1 - while cls.__counter__: + checked, second_waited = 0, 0 + checked = 0 + while True: + time.sleep(wait_for) + second_waited += wait_for if second_waited > max_wait: logger.warn('Exiting after %d seconds, counter at %d', - max_wait, cls.__counter__) + max_wait, len(cls.__counter__)) break - time.sleep(1) - second_waited += 1 + if cls.get_session().executor._work_queue.queue: + checked = 0 + continue + checked += 1 + if checked == checks: + break class PyAggUpdater(AbstractCrawler): @@ -109,13 +89,11 @@ class PyAggUpdater(AbstractCrawler): self.entries = entries self.headers = headers self.parsed_feed = parsed_feed - super(PyAggUpdater, self).__init__(auth) + super().__init__(auth) - @AbstractCrawler.count_on_me def callback(self, response): """Will process the result from the challenge, creating missing article and updating the feed""" - AbstractCrawler.__counter__ -= 1 article_created = False if response.result().status_code != 204: results = response.result().json() @@ -162,27 +140,23 @@ class PyAggUpdater(AbstractCrawler): future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed) - future.add_done_callback(self.get_counter_callback()) class FeedCrawler(AbstractCrawler): def __init__(self, feed, auth): self.feed = feed - super(FeedCrawler, self).__init__(auth) + super().__init__(auth) def clean_feed(self): """Will reset the errors counters on a feed that have known errors""" if self.feed.get('error_count') or self.feed.get('last_error'): future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': 0, 'last_error': ''}) - future.add_done_callback(self.get_counter_callback()) - @AbstractCrawler.count_on_me def callback(self, response): """will fetch the feed and interprete results (304, etag) or will challenge pyagg to compare gotten entries with existing ones""" - AbstractCrawler.__counter__ -= 1 try: response = response.result() response.raise_for_status() @@ -194,7 +168,6 @@ class FeedCrawler(AbstractCrawler): future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': error_count, 'last_error': str(error)}) - future.add_done_callback(self.get_counter_callback()) return if response.status_code == 304: @@ -246,7 +219,6 @@ class CrawlerScheduler(AbstractCrawler): def __init__(self, username, password): self.auth = (username, password) super(CrawlerScheduler, self).__init__(self.auth) - AbstractCrawler.__counter__ = 0 def prepare_headers(self, feed): """For a known feed, will construct some header dictionnary""" @@ -259,10 +231,8 @@ class CrawlerScheduler(AbstractCrawler): feed['id'], feed['title'], headers) return headers - @AbstractCrawler.count_on_me def callback(self, response): """processes feeds that need to be fetched""" - AbstractCrawler.__counter__ -= 1 response = response.result() response.raise_for_status() if response.status_code == 204: @@ -277,11 +247,9 @@ class CrawlerScheduler(AbstractCrawler): headers=self.prepare_headers(feed)) future.add_done_callback(FeedCrawler(feed, self.auth).callback) - @AbstractCrawler.count_on_me def run(self, **kwargs): """entry point, will retreive feeds to be fetch and launch the whole thing""" logger.debug('retreving fetchable feed') future = self.query_pyagg('get', 'feeds/fetchable', kwargs) - AbstractCrawler.__counter__ += 1 future.add_done_callback(self.callback) -- cgit From d770bc17899b7e947252237e04a71075999c7f59 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Tue, 4 Aug 2015 18:17:10 +0200 Subject: refact feed list --- pyaggr3g470r/templates/admin/user.html | 35 +------------------------ pyaggr3g470r/templates/feed_list.html | 47 ++++++++++++++++++++++++++++++++++ pyaggr3g470r/templates/feeds.html | 44 +------------------------------ pyaggr3g470r/views/views.py | 17 +++++++----- 4 files changed, 59 insertions(+), 84 deletions(-) create mode 100644 pyaggr3g470r/templates/feed_list.html diff --git a/pyaggr3g470r/templates/admin/user.html b/pyaggr3g470r/templates/admin/user.html index 21bcd6b6..d1e08c0d 100644 --- a/pyaggr3g470r/templates/admin/user.html +++ b/pyaggr3g470r/templates/admin/user.html @@ -14,39 +14,6 @@ -
- {% if user.feeds.all()|count == 0 %} -

{{ _('This user is not subscribed to any feed.') }}

- {% else %} -

{{ _('Feeds') }}

- - - - - - - - - - - - - {% for feed in user.feeds|sort(attribute="title") %} - - - - - - - - {% endfor %} - -
#{{ _('Title') }}{{ _('Feed link') }}{{ _('Site link') }}{{ _('(unread) articles') }}{{ _('Actions') }}
{{ loop.index }}{%if feed.icon%}{%endif%}{{ feed.title }}{{ feed.link }}{{ feed.site_link }}( {{ unread_article_count.get(feed.id, 0) }} ) {{ article_count.get(feed.id, 0) }} - - - -
- {% endif %} -
+ {% include "feed_list.html" %} {% endblock %} diff --git a/pyaggr3g470r/templates/feed_list.html b/pyaggr3g470r/templates/feed_list.html new file mode 100644 index 00000000..6ef612b4 --- /dev/null +++ b/pyaggr3g470r/templates/feed_list.html @@ -0,0 +1,47 @@ +{% if feeds.all()| count == 0 %} +

{{_("No feed")}}

+{% else %} +
+ + + + + + + + + + + + + {% for feed in feeds|sort(attribute="title") %} + + + + + + + + + {% endfor %} + +
#{{ _('Status') }}{{ _('Title') }}{{ _('Site') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }} + {% if feed.enabled %} + + {% else %} + + {% endif %} + {% if feed.error_count > conf.DEFAULT_MAX_ERROR %} + + {% endif %} + + {% if feed.icon_url %}{% endif %} + {{ feed.title }} + {{ feed.site_link }}( {{ unread_article_count.get(feed.id, 0) }} ) {{ article_count.get(feed.id, 0) }} + + + + +
+
+{% endif %} diff --git a/pyaggr3g470r/templates/feeds.html b/pyaggr3g470r/templates/feeds.html index 82af2411..9ba16359 100644 --- a/pyaggr3g470r/templates/feeds.html +++ b/pyaggr3g470r/templates/feeds.html @@ -2,48 +2,6 @@ {% block content %}

{{ _('You are subscribed to') }} {{ feeds.count() }} {{ _('feeds') }} · {{ _('Add a') }} {{ _('feed') }}

-
- - - - - - - - - - - - - {% for feed in feeds|sort(attribute="title") %} - - - - - - - - - {% endfor %} - -
#{{ _('Status') }}{{ _('Title') }}{{ _('Site') }}{{ _('Articles') }}{{ _('Actions') }}
{{ loop.index }} - {% if feed.enabled %} - - {% else %} - - {% endif %} - {% if feed.error_count > conf.DEFAULT_MAX_ERROR %} - - {% endif %} - - {% if feed.icon_url %}{% endif %} - {{ feed.title }} - {{ feed.site_link }}( {{ unread_article_count.get(feed.id, 0) }} ) {{ article_count.get(feed.id, 0) }} - - - - -
-
+ {% include "feed_list.html" %}
{% endblock %} diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index 69c2b50b..560bd382 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -718,8 +718,10 @@ def create_user(user_id=None): user.activation_key = "" db.session.add(user) db.session.commit() - flash(gettext('User') + ' ' + user.nickname + ' ' + gettext('successfully created.'), 'success') - return redirect("/admin/edit_user/"+str(user.id)) + flash("%s %s %s" % (gettext('User'), user.nickname, + gettext('successfully created.')), + 'success') + return redirect(url_for('create_user', user_id=user.id)) else: return redirect(url_for('create_user')) @@ -727,11 +729,13 @@ def create_user(user_id=None): if user_id is not None: user = User.query.filter(User.id == user_id).first() form = UserForm(obj=user) - message = gettext('Edit the user') + ' ' + user.nickname + '' + message = "%s %s" % (gettext('Edit the user'), + user.nickname) else: form = UserForm() message = gettext('Add a new user') - return render_template('/admin/create_user.html', form=form, message=message) + return render_template('/admin/create_user.html', + form=form, message=message) @app.route('/admin/user/', methods=['GET']) @login_required @@ -743,10 +747,9 @@ def user(user_id=None): user = UserController().get(id=user_id) if user is not None: article_contr = ArticleController(user_id) - return render_template('/admin/user.html', user=user, + return render_template('/admin/user.html', user=user, feeds=user.feeds, article_count=article_contr.count_by_feed(), - unread_article_count=article_contr.count_by_feed(readed=False), - ) + unread_article_count=article_contr.count_by_feed(readed=False)) else: flash(gettext('This user does not exist.'), 'danger') -- cgit From 5e0aca072dc5ecb6a041f996e3b4ad5ef6d1205c Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Wed, 5 Aug 2015 16:05:29 +0200 Subject: making it easier to request with lists --- pyaggr3g470r/views/api/common.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyaggr3g470r/views/api/common.py b/pyaggr3g470r/views/api/common.py index a7068807..acb5dd68 100644 --- a/pyaggr3g470r/views/api/common.py +++ b/pyaggr3g470r/views/api/common.py @@ -161,17 +161,17 @@ class PyAggResourceMulti(PyAggAbstractResource): """retrieve several objects. filters can be set in the payload on the different fields of the object, and a limit can be set in there as well """ - if 'application/json' not in request.headers.get('Content-Type'): - raise BadRequest("Content-Type must be application/json") - limit = 10 try: limit = request.json.pop('limit', 10) + order_by = request.json.pop('order_by', None) + query = self.controller.read(**request.json) except: - return [res for res in self.controller.read().limit(limit)] - if not limit: - return [res for res in self.controller.read(**request.json).all()] - return [res - for res in self.controller.read(**request.json).limit(limit)] + limit, order_by, query = 10, None, self.controller.read() + if order_by: + query = query.order_by(order_by) + if limit: + query = query.limit(limit) + return [res for res in query] def post(self): """creating several objects. payload should be a list of dict. -- cgit From afcfb5cf3eef2e2b15cc85dd4d4485c994045d14 Mon Sep 17 00:00:00 2001 From: François Schmidts Date: Wed, 5 Aug 2015 17:25:51 +0200 Subject: fixing bug preventing from bumping error count from api crawler for none admin user --- pyaggr3g470r/lib/crawler.py | 3 ++- pyaggr3g470r/templates/feed_list.html | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index b599e2d4..91942c59 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -167,7 +167,8 @@ class FeedCrawler(AbstractCrawler): self.feed['title'], error_count) future = self.query_pyagg('put', 'feed/%d' % self.feed['id'], {'error_count': error_count, - 'last_error': str(error)}) + 'last_error': str(error), + 'user_id': self.feed['user_id']}) return if response.status_code == 304: diff --git a/pyaggr3g470r/templates/feed_list.html b/pyaggr3g470r/templates/feed_list.html index 6ef612b4..c5cadab0 100644 --- a/pyaggr3g470r/templates/feed_list.html +++ b/pyaggr3g470r/templates/feed_list.html @@ -23,7 +23,7 @@ {% else %} {% endif %} - {% if feed.error_count > conf.DEFAULT_MAX_ERROR %} + {% if feed.error_count >= conf.DEFAULT_MAX_ERROR %} {% endif %} -- cgit