diff options
| author | Cédric Bonhomme <kimble.mandel@gmail.com> | 2013-08-11 14:13:31 +0200 |
|---|---|---|
| committer | Cédric Bonhomme <kimble.mandel@gmail.com> | 2013-08-11 14:13:31 +0200 |
| commit | c42aa9a1e2ec1a9afc0f6a3fbbc402a4d05ec4f7 (patch) | |
| tree | 020b9eae53956ff1d9daa1ed97ef554f5115c73a /source/feedgetter.py | |
| parent | The /management page dislays the number of indexed documents. (diff) | |
| download | newspipe-c42aa9a1e2ec1a9afc0f6a3fbbc402a4d05ec4f7.tar.gz newspipe-c42aa9a1e2ec1a9afc0f6a3fbbc402a4d05ec4f7.tar.bz2 newspipe-c42aa9a1e2ec1a9afc0f6a3fbbc402a4d05ec4f7.zip | |
It is now possible to fetch articles with feedparser through a HTTP proxy, for example privoxy/tor. The address of the proxy is specified in the configuration file.
Diffstat (limited to 'source/feedgetter.py')
| -rwxr-xr-x | source/feedgetter.py | 11 |
1 files changed, 8 insertions, 3 deletions
diff --git a/source/feedgetter.py b/source/feedgetter.py index 44fd7daa..cec74938 100755 --- a/source/feedgetter.py +++ b/source/feedgetter.py @@ -20,14 +20,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/> __author__ = "Cedric Bonhomme" -__version__ = "$Revision: 1.6 $" +__version__ = "$Revision: 1.7 $" __date__ = "$Date: 2010/09/02 $" -__revision__ = "$Date: 2013/06/10 $" +__revision__ = "$Date: 2013/08/11 $" __copyright__ = "Copyright (c) Cedric Bonhomme" __license__ = "GPLv3" import hashlib import threading +import urllib.request import feedparser from bs4 import BeautifulSoup from datetime import datetime @@ -109,7 +110,11 @@ class FeedGetter(object): """ Add the articles of the feed 'a_feed' in the SQLite base. """ - a_feed = feedparser.parse(feed_link) + if conf.HTTP_PROXY == "": + proxy = urllib.request.ProxyHandler({}) + else: + proxy = urllib.request.ProxyHandler({"http":conf.HTTP_PROXY}) + a_feed = feedparser.parse(feed_link, handlers = [proxy]) if a_feed['entries'] == []: return try: |