From b733818e2ae53ecac926a764c575db4037c5a8af Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Sun, 19 Jan 2014 11:30:44 +0100 Subject: Only get the URL with requests if the string 'feedproxy' is found in the netloc part of the original URL. --- pyaggr3g470r/feedgetter.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'pyaggr3g470r') diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index 7bf87fea..da28663c 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -108,16 +108,17 @@ class FeedGetter(object): for article in a_feed['entries']: nice_url = article.link.encode("utf-8") - try: - # resolves URL behind proxies (like feedproxy.google.com) - r = requests.get(article.link, timeout=10.0, proxies=self.proxies) - nice_url = r.url.encode("utf-8") - except Timeout: - pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,)) - continue - except Exception as e: - pyaggr3g470r_log.warning("Unable to get the real URL of %s. Error: %s" % (article.link, str(e))) - continue + if "feedproxy" in urlparse(nice_url).netloc: + try: + # resolves URL behind proxies (like feedproxy.google.com) + r = requests.get(article.link, timeout=10.0, proxies=self.proxies) + nice_url = r.url.encode("utf-8") + except Timeout: + pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,)) + continue + except Exception as e: + pyaggr3g470r_log.warning("Unable to get the real URL of %s. Error: %s" % (article.link, str(e))) + continue # remove utm_* parameters parsed_url = urlparse(nice_url) qd = parse_qs(parsed_url.query, keep_blank_values=True) -- cgit