From d52f5735c6e540d30e716939fa619aedbd9d3aeb Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Tue, 24 Dec 2013 11:54:54 +0100 Subject: It is now possible to configure the proxy and the user agent. --- conf.py | 3 +++ pyaggr3g470r/feedgetter.py | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/conf.py b/conf.py index d834ea38..3f406477 100644 --- a/conf.py +++ b/conf.py @@ -23,6 +23,9 @@ DATABASE_USERNAME = config.get('database', 'username') DATABASE_PASSWORD = config.get('database', 'password') DATABASE_ADDRESS = config.get('database', 'address') +HTTP_PROXY = config.get('feedparser', 'http_proxy') +USER_AGENT = config.get('feedparser', 'user_agent') + WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1 WEBSERVER_HOST = config.get('webserver', 'host') WEBSERVER_PORT = int(config.get('webserver', 'port')) diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index 673b5a20..49467761 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -27,7 +27,7 @@ __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" import threading - +import urllib2 import feedparser from BeautifulSoup import BeautifulSoup from datetime import datetime @@ -57,6 +57,11 @@ class FeedGetter(object): """ #feedparser.USER_AGENT = conf.USER_AGENT feedparser.USER_AGENT = "pyAggr3g470r" + if conf.HTTP_PROXY == "": + self.proxy = urllib2.ProxyHandler({}) + else: + self.proxy = urllib2.ProxyHandler({"http" : conf.HTTP_PROXY}) + feedparser.USER_AGENT = conf.USER_AGENT self.user = models.User.objects(email=email).first() def retrieve_feed(self): @@ -81,8 +86,7 @@ class FeedGetter(object): """ Retrieves articles form the feed and add them to the database. """ - #a_feed = feedparser.parse(feed_link, handlers = [self.proxy]) - a_feed = feedparser.parse(feed.link) + a_feed = feedparser.parse(feed.link, handlers = [self.proxy]) if a_feed['entries'] == []: return -- cgit