diff options
| -rw-r--r-- | conf.py | 3 | ||||
| -rw-r--r-- | pyaggr3g470r/feedgetter.py | 10 |
2 files changed, 10 insertions, 3 deletions
@@ -23,6 +23,9 @@ DATABASE_USERNAME = config.get('database', 'username') DATABASE_PASSWORD = config.get('database', 'password') DATABASE_ADDRESS = config.get('database', 'address') +HTTP_PROXY = config.get('feedparser', 'http_proxy') +USER_AGENT = config.get('feedparser', 'user_agent') + WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1 WEBSERVER_HOST = config.get('webserver', 'host') WEBSERVER_PORT = int(config.get('webserver', 'port')) diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index 673b5a20..49467761 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -27,7 +27,7 @@ __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" import threading - +import urllib2 import feedparser from BeautifulSoup import BeautifulSoup from datetime import datetime @@ -57,6 +57,11 @@ class FeedGetter(object): """ #feedparser.USER_AGENT = conf.USER_AGENT feedparser.USER_AGENT = "pyAggr3g470r" + if conf.HTTP_PROXY == "": + self.proxy = urllib2.ProxyHandler({}) + else: + self.proxy = urllib2.ProxyHandler({"http" : conf.HTTP_PROXY}) + feedparser.USER_AGENT = conf.USER_AGENT self.user = models.User.objects(email=email).first() def retrieve_feed(self): @@ -81,8 +86,7 @@ class FeedGetter(object): """ Retrieves articles form the feed and add them to the database. """ - #a_feed = feedparser.parse(feed_link, handlers = [self.proxy]) - a_feed = feedparser.parse(feed.link) + a_feed = feedparser.parse(feed.link, handlers = [self.proxy]) if a_feed['entries'] == []: return |