1 files changed, 11 insertions, 0 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py
new file mode 100644
index 00000000..1d7fca71
--- /dev/null
+++ b/pyaggr3g470r/lib/crawler.py
@@ -0,0 +1,11 @@
+import feedparser
+import dateutil.parser.parse
+
+
+def get_feed_content(feed):
+    etag = feed.get('etag', None)
+    last_modified = None
+    if feed.get('last_modified'):
+        last_modified = dateutil.parser.parse(feed['last_modified'])\
+                                    .strftime('%a, %d %b %Y %H:%M:%S %Z')
+    return feedparser.parse(feed['link'], etag=etag, modified=last_modified)