diff options
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/lib/feed_utils.py | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/lib/utils.py | 2 | ||||
| -rw-r--r-- | pyaggr3g470r/templates/layout.html | 18 | ||||
| -rw-r--r-- | pyaggr3g470r/views/feed.py | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/views/views.py | 11 |
6 files changed, 21 insertions, 31 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py index de557e45..e5998776 100644 --- a/pyaggr3g470r/lib/crawler.py +++ b/pyaggr3g470r/lib/crawler.py @@ -17,6 +17,7 @@ import conf import json import logging import feedparser +from datetime import datetime, timedelta from functools import wraps from time import strftime, gmtime from concurrent.futures import ThreadPoolExecutor @@ -118,7 +119,9 @@ class PyAggUpdater(AbstractCrawler): results = response.result().json() logger.debug('%r %r - %d entries were not matched and will be created', self.feed['id'], self.feed['title'], len(results)) + article_created = False for id_to_create in results: + article_created = True entry = construct_article( self.entries[tuple(sorted(id_to_create.items()))], self.feed) @@ -144,6 +147,10 @@ class PyAggUpdater(AbstractCrawler): if not self.feed.get('title'): up_feed['title'] = fresh_feed.get('title', '') up_feed['user_id'] = self.feed['user_id'] + # re-getting that feed earlier since new entries appeared + if article_created: + up_feed['last_retrieved'] \ + = (datetime.now() - timedelta(minutes=45)).isoformat() logger.info('%r %r - pushing feed attrs %r', self.feed['id'], self.feed['title'], diff --git a/pyaggr3g470r/lib/feed_utils.py b/pyaggr3g470r/lib/feed_utils.py index a7149d79..367fd4b5 100644 --- a/pyaggr3g470r/lib/feed_utils.py +++ b/pyaggr3g470r/lib/feed_utils.py @@ -23,6 +23,7 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): assert url is not None and fp_parsed is not None feed = feed or {} feed_split = urllib.parse.urlsplit(url) + site_split = None if not fp_parsed['bozo']: feed['link'] = url feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link') @@ -82,8 +83,8 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): del feed['icon'] if not feed.get('link'): - alternate = bs_parsed.find_all(check_keys(rel=['alternate'], + alternates = bs_parsed.find_all(check_keys(rel=['alternate'], type=['application/rss+xml'])) - if len(alternate) >= 1: - feed['link'] = alternate[0].attrs['href'] + if len(alternates) >= 1: + feed['link'] = rebuild_url(alternates[0].attrs['href'], feed_split) return feed diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py index 280256f6..62284de1 100644 --- a/pyaggr3g470r/lib/utils.py +++ b/pyaggr3g470r/lib/utils.py @@ -42,6 +42,8 @@ def rebuild_url(url, base_split): def try_splits(url, *splits): for split in splits: + if split is None: + continue rb_url = rebuild_url(url, split) response = requests.get(rb_url, verify=False, timeout=10) if response.ok and 'html' not in response.headers['content-type']: diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html index fcdbee32..b37fb3e1 100644 --- a/pyaggr3g470r/templates/layout.html +++ b/pyaggr3g470r/templates/layout.html @@ -45,22 +45,8 @@ </a> <ul class="dropdown-menu"> <li> - <form action="/feed/create" class="navbar-form" method="POST" name="save"> - {{ create_feed_form.hidden_tag() }} - - {{ create_feed_form.link.label }} - {{ create_feed_form.link(class_="form-control") }} {% for error in create_feed_form.link.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %} - - {{ create_feed_form.title.label }} - {{ create_feed_form.title(class_="form-control", placeholder=_('Optional')) }} {% for error in create_feed_form.title.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %} - - {{ create_feed_form.site_link.label }} - {{ create_feed_form.site_link(class_="form-control", placeholder=_('Optional')) }} {% for error in create_feed_form.site_link.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %} - - {{ create_feed_form.enabled.label }} - {{ create_feed_form.enabled(class_="checkbox") }} - <br /> - {{ create_feed_form.submit(class_="btn btn-default") }} + <form action="{{ url_for('feed.bookmarklet') }}" class="navbar-form" method="GET" name="save"> + <input class="form-control" name="url" type="text" placeholder="{{_("site or feed url")}}" /> <input type="submit" value="+" class="btn btn-default" /> </form> </li> </ul> diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py index 18e110af..86d07e18 100644 --- a/pyaggr3g470r/views/feed.py +++ b/pyaggr3g470r/views/feed.py @@ -102,7 +102,12 @@ def bookmarklet(): "warning") return redirect(url_for('feed.form', feed_id=feed_exists[0].id)) - feed = feed_contr.create(**construct_feed_from(url)) + feed = construct_feed_from(url) + if not feed.get('link'): + feed['enabled'] = False + flash(gettext("Couldn't find a feed url, you'll need to find a Atom or" + "RSS link manually and reactivate this feed"), 'warning') + feed = feed_contr.create(**feed) flash(gettext('Feed was successfully created.'), 'success') if conf.CRAWLING_METHOD == "classic": utils.fetch(g.user.id, feed.id) diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py index a4e799cc..189fd53f 100644 --- a/pyaggr3g470r/views/views.py +++ b/pyaggr3g470r/views/views.py @@ -137,17 +137,6 @@ def get_timezone(): except: return conf.TIME_ZONE["en"] - -@app.context_processor -def inject_feed_form(): - """ - Injects the 'AddFeedForm' objects in all templates. - - Context processors run before the template is rendered and have the - ability to inject new values into the template context. - """ - return dict(create_feed_form=AddFeedForm()) - # # Views. # |