diff options
| author | Cédric Bonhomme <kimble.mandel@gmail.com> | 2014-01-20 07:48:26 +0100 |
|---|---|---|
| committer | Cédric Bonhomme <kimble.mandel@gmail.com> | 2014-01-20 07:48:26 +0100 |
| commit | 9f18772bf22c2bca731ab9e350e1cb283252eec5 (patch) | |
| tree | 0813d861823cec60a3bfcd71b8d1f06f37ee9b5b /pyaggr3g470r | |
| parent | The MongoEngine object is saved after all threads are done. (diff) | |
| download | newspipe-9f18772bf22c2bca731ab9e350e1cb283252eec5.tar.gz newspipe-9f18772bf22c2bca731ab9e350e1cb283252eec5.tar.bz2 newspipe-9f18772bf22c2bca731ab9e350e1cb283252eec5.zip | |
Finally always try to get the real URL...
Diffstat (limited to 'pyaggr3g470r')
| -rw-r--r-- | pyaggr3g470r/feedgetter.py | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index cc9f8aa4..3eb69938 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -110,18 +110,17 @@ class FeedGetter(object): for article in a_feed['entries']: nice_url = article.link.encode("utf-8") - if "feedproxy" in urlparse(nice_url).netloc: - try: - # resolves URL behind proxies (like feedproxy.google.com) - r = requests.get(article.link, timeout=5.0, proxies=self.proxies) - nice_url = r.url.encode("utf-8") - except Timeout: - pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,)) - print "Time out" - continue - except Exception as e: - pyaggr3g470r_log.warning("Unable to get the real URL of %s. Error: %s" % (article.link, str(e))) - continue + try: + # resolves URL behind proxies (like feedproxy.google.com) + r = requests.get(article.link, timeout=5.0, proxies=self.proxies) + nice_url = r.url.encode("utf-8") + except Timeout: + pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,)) + print "Time out" + continue + except Exception as e: + pyaggr3g470r_log.warning("Unable to get the real URL of %s. Error: %s" % (article.link, str(e))) + continue # remove utm_* parameters parsed_url = urlparse(nice_url) qd = parse_qs(parsed_url.query, keep_blank_values=True) |