aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <cedric@cedricbonhomme.org>2013-12-24 11:54:54 +0100
committerCédric Bonhomme <cedric@cedricbonhomme.org>2013-12-24 11:54:54 +0100
commitd52f5735c6e540d30e716939fa619aedbd9d3aeb (patch)
treedfb02bff4e23246f1767bfb70ac4fb42aae7403e
parentUpdated default configuration file. (diff)
downloadnewspipe-d52f5735c6e540d30e716939fa619aedbd9d3aeb.tar.gz
newspipe-d52f5735c6e540d30e716939fa619aedbd9d3aeb.tar.bz2
newspipe-d52f5735c6e540d30e716939fa619aedbd9d3aeb.zip
It is now possible to configure the proxy and the user agent.
-rw-r--r--conf.py3
-rw-r--r--pyaggr3g470r/feedgetter.py10
2 files changed, 10 insertions, 3 deletions
diff --git a/conf.py b/conf.py
index d834ea38..3f406477 100644
--- a/conf.py
+++ b/conf.py
@@ -23,6 +23,9 @@ DATABASE_USERNAME = config.get('database', 'username')
DATABASE_PASSWORD = config.get('database', 'password')
DATABASE_ADDRESS = config.get('database', 'address')
+HTTP_PROXY = config.get('feedparser', 'http_proxy')
+USER_AGENT = config.get('feedparser', 'user_agent')
+
WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1
WEBSERVER_HOST = config.get('webserver', 'host')
WEBSERVER_PORT = int(config.get('webserver', 'port'))
diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py
index 673b5a20..49467761 100644
--- a/pyaggr3g470r/feedgetter.py
+++ b/pyaggr3g470r/feedgetter.py
@@ -27,7 +27,7 @@ __copyright__ = "Copyright (c) Cedric Bonhomme"
__license__ = "GPLv3"
import threading
-
+import urllib2
import feedparser
from BeautifulSoup import BeautifulSoup
from datetime import datetime
@@ -57,6 +57,11 @@ class FeedGetter(object):
"""
#feedparser.USER_AGENT = conf.USER_AGENT
feedparser.USER_AGENT = "pyAggr3g470r"
+ if conf.HTTP_PROXY == "":
+ self.proxy = urllib2.ProxyHandler({})
+ else:
+ self.proxy = urllib2.ProxyHandler({"http" : conf.HTTP_PROXY})
+ feedparser.USER_AGENT = conf.USER_AGENT
self.user = models.User.objects(email=email).first()
def retrieve_feed(self):
@@ -81,8 +86,7 @@ class FeedGetter(object):
"""
Retrieves articles form the feed and add them to the database.
"""
- #a_feed = feedparser.parse(feed_link, handlers = [self.proxy])
- a_feed = feedparser.parse(feed.link)
+ a_feed = feedparser.parse(feed.link, handlers = [self.proxy])
if a_feed['entries'] == []:
return
bgstack15