aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pyaggr3g470r/lib/crawler.py7
-rw-r--r--pyaggr3g470r/lib/feed_utils.py7
-rw-r--r--pyaggr3g470r/lib/utils.py2
-rw-r--r--pyaggr3g470r/templates/layout.html18
-rw-r--r--pyaggr3g470r/views/feed.py7
-rw-r--r--pyaggr3g470r/views/views.py11
6 files changed, 21 insertions, 31 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
index de557e45..e5998776 100644
--- a/pyaggr3g470r/lib/crawler.py
+++ b/pyaggr3g470r/lib/crawler.py
@@ -17,6 +17,7 @@ import conf
import json
import logging
import feedparser
+from datetime import datetime, timedelta
from functools import wraps
from time import strftime, gmtime
from concurrent.futures import ThreadPoolExecutor
@@ -118,7 +119,9 @@ class PyAggUpdater(AbstractCrawler):
results = response.result().json()
logger.debug('%r %r - %d entries were not matched and will be created',
self.feed['id'], self.feed['title'], len(results))
+ article_created = False
for id_to_create in results:
+ article_created = True
entry = construct_article(
self.entries[tuple(sorted(id_to_create.items()))],
self.feed)
@@ -144,6 +147,10 @@ class PyAggUpdater(AbstractCrawler):
if not self.feed.get('title'):
up_feed['title'] = fresh_feed.get('title', '')
up_feed['user_id'] = self.feed['user_id']
+ # re-getting that feed earlier since new entries appeared
+ if article_created:
+ up_feed['last_retrieved'] \
+ = (datetime.now() - timedelta(minutes=45)).isoformat()
logger.info('%r %r - pushing feed attrs %r',
self.feed['id'], self.feed['title'],
diff --git a/pyaggr3g470r/lib/feed_utils.py b/pyaggr3g470r/lib/feed_utils.py
index a7149d79..367fd4b5 100644
--- a/pyaggr3g470r/lib/feed_utils.py
+++ b/pyaggr3g470r/lib/feed_utils.py
@@ -23,6 +23,7 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
assert url is not None and fp_parsed is not None
feed = feed or {}
feed_split = urllib.parse.urlsplit(url)
+ site_split = None
if not fp_parsed['bozo']:
feed['link'] = url
feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link')
@@ -82,8 +83,8 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True):
del feed['icon']
if not feed.get('link'):
- alternate = bs_parsed.find_all(check_keys(rel=['alternate'],
+ alternates = bs_parsed.find_all(check_keys(rel=['alternate'],
type=['application/rss+xml']))
- if len(alternate) >= 1:
- feed['link'] = alternate[0].attrs['href']
+ if len(alternates) >= 1:
+ feed['link'] = rebuild_url(alternates[0].attrs['href'], feed_split)
return feed
diff --git a/pyaggr3g470r/lib/utils.py b/pyaggr3g470r/lib/utils.py
index 280256f6..62284de1 100644
--- a/pyaggr3g470r/lib/utils.py
+++ b/pyaggr3g470r/lib/utils.py
@@ -42,6 +42,8 @@ def rebuild_url(url, base_split):
def try_splits(url, *splits):
for split in splits:
+ if split is None:
+ continue
rb_url = rebuild_url(url, split)
response = requests.get(rb_url, verify=False, timeout=10)
if response.ok and 'html' not in response.headers['content-type']:
diff --git a/pyaggr3g470r/templates/layout.html b/pyaggr3g470r/templates/layout.html
index fcdbee32..b37fb3e1 100644
--- a/pyaggr3g470r/templates/layout.html
+++ b/pyaggr3g470r/templates/layout.html
@@ -45,22 +45,8 @@
</a>
<ul class="dropdown-menu">
<li>
- <form action="/feed/create" class="navbar-form" method="POST" name="save">
- {{ create_feed_form.hidden_tag() }}
-
- {{ create_feed_form.link.label }}
- {{ create_feed_form.link(class_="form-control") }} {% for error in create_feed_form.link.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
-
- {{ create_feed_form.title.label }}
- {{ create_feed_form.title(class_="form-control", placeholder=_('Optional')) }} {% for error in create_feed_form.title.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
-
- {{ create_feed_form.site_link.label }}
- {{ create_feed_form.site_link(class_="form-control", placeholder=_('Optional')) }} {% for error in create_feed_form.site_link.errors %} <span style="color: red;">{{ error }}<br /></span>{% endfor %}
-
- {{ create_feed_form.enabled.label }}
- {{ create_feed_form.enabled(class_="checkbox") }}
- <br />
- {{ create_feed_form.submit(class_="btn btn-default") }}
+ <form action="{{ url_for('feed.bookmarklet') }}" class="navbar-form" method="GET" name="save">
+ <input class="form-control" name="url" type="text" placeholder="{{_("site or feed url")}}" /> <input type="submit" value="+" class="btn btn-default" />
</form>
</li>
</ul>
diff --git a/pyaggr3g470r/views/feed.py b/pyaggr3g470r/views/feed.py
index 18e110af..86d07e18 100644
--- a/pyaggr3g470r/views/feed.py
+++ b/pyaggr3g470r/views/feed.py
@@ -102,7 +102,12 @@ def bookmarklet():
"warning")
return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
- feed = feed_contr.create(**construct_feed_from(url))
+ feed = construct_feed_from(url)
+ if not feed.get('link'):
+ feed['enabled'] = False
+ flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
+ "RSS link manually and reactivate this feed"), 'warning')
+ feed = feed_contr.create(**feed)
flash(gettext('Feed was successfully created.'), 'success')
if conf.CRAWLING_METHOD == "classic":
utils.fetch(g.user.id, feed.id)
diff --git a/pyaggr3g470r/views/views.py b/pyaggr3g470r/views/views.py
index a4e799cc..189fd53f 100644
--- a/pyaggr3g470r/views/views.py
+++ b/pyaggr3g470r/views/views.py
@@ -137,17 +137,6 @@ def get_timezone():
except:
return conf.TIME_ZONE["en"]
-
-@app.context_processor
-def inject_feed_form():
- """
- Injects the 'AddFeedForm' objects in all templates.
-
- Context processors run before the template is rendered and have the
- ability to inject new values into the template context.
- """
- return dict(create_feed_form=AddFeedForm())
-
#
# Views.
#
bgstack15