diff options
Diffstat (limited to 'pyaggr3g470r/lib/crawler.py')
| -rw-r--r-- | pyaggr3g470r/lib/crawler.py | 11 |
1 files changed, 11 insertions, 0 deletions
diff --git a/pyaggr3g470r/lib/crawler.py b/pyaggr3g470r/lib/crawler.py new file mode 100644 index 00000000..1d7fca71 --- /dev/null +++ b/pyaggr3g470r/lib/crawler.py @@ -0,0 +1,11 @@ +import feedparser +import dateutil.parser.parse + + +def get_feed_content(feed): + etag = feed.get('etag', None) + last_modified = None + if feed.get('last_modified'): + last_modified = dateutil.parser.parse(feed['last_modified'])\ + .strftime('%a, %d %b %Y %H:%M:%S %Z') + return feedparser.parse(feed['link'], etag=etag, modified=last_modified) |