diff options
| -rw-r--r-- | pyaggr3g470r/crawler.py | 20 | ||||
| -rw-r--r-- | pyaggr3g470r/duplicate.py | 2 | ||||
| -rw-r--r-- | pyaggr3g470r/notifications.py | 4 |
3 files changed, 16 insertions, 10 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index 6472e35d..2e07e844 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -30,7 +30,12 @@ import re import socket import logging import feedparser -import urllib2 +try: + # Python 3 + from urllib.request import ProxyHandler +except: + # Python 2 + from urllib2 import ProxyHandler import requests import dateutil.parser from bs4 import BeautifulSoup @@ -43,13 +48,13 @@ gevent.monkey.patch_all() from gevent import Timeout from gevent.pool import Pool -import utils -import conf -import notifications +from pyaggr3g470r import utils +from pyaggr3g470r import conf +from pyaggr3g470r import notifications from pyaggr3g470r import db from pyaggr3g470r.models import User, Article if not conf.ON_HEROKU: - import search as fastsearch + import pyaggr3g470r.search as fastsearch logger = logging.getLogger(__name__) @@ -74,10 +79,10 @@ class FeedGetter(object): """ feedparser.USER_AGENT = conf.USER_AGENT if conf.HTTP_PROXY == "": - self.proxy = urllib2.ProxyHandler({}) + self.proxy = ProxyHandler({}) self.proxies = {} else: - self.proxy = urllib2.ProxyHandler({"http": conf.HTTP_PROXY, + self.proxy = ProxyHandler({"http": conf.HTTP_PROXY, "https": conf.HTTP_PROXY}) self.proxies = { "http": "http://" + conf.HTTP_PROXY, @@ -128,6 +133,7 @@ class FeedGetter(object): """ logger.info("Fetching the feed: " + feed.title) a_feed = feedparser.parse(feed.link, handlers=[self.proxy]) + print(a_feed) if a_feed['bozo'] == 1: logger.error(a_feed['bozo_exception']) if a_feed['entries'] == []: diff --git a/pyaggr3g470r/duplicate.py b/pyaggr3g470r/duplicate.py index f34e284e..d4c6e31a 100644 --- a/pyaggr3g470r/duplicate.py +++ b/pyaggr3g470r/duplicate.py @@ -4,7 +4,7 @@ import itertools from datetime import timedelta -import utils +from pyaggr3g470r import utils def compare_documents(feed): """ diff --git a/pyaggr3g470r/notifications.py b/pyaggr3g470r/notifications.py index 94f34774..34548f3c 100644 --- a/pyaggr3g470r/notifications.py +++ b/pyaggr3g470r/notifications.py @@ -19,8 +19,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import utils -import conf +from pyaggr3g470r import utils +from pyaggr3g470r import conf from pyaggr3g470r import emails def information_message(subject, plaintext): |