From cb978e1bdd966710e23a2817e8d3b6124995e4e8 Mon Sep 17 00:00:00 2001
From: Cédric Bonhomme <kimble.mandel@gmail.com>
Date: Mon, 6 Jan 2014 15:28:32 +0100
Subject: utm_* parameters are automatically removed from article's URLs.

---
 pyaggr3g470r/feedgetter.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

(limited to 'pyaggr3g470r')

diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py
index 55564840..9c6c240e 100644
--- a/pyaggr3g470r/feedgetter.py
+++ b/pyaggr3g470r/feedgetter.py
@@ -31,7 +31,8 @@ import requests
 import threading
 import feedparser
 from datetime import datetime
-from urlparse import urlparse
+from urllib import urlencode
+from urlparse import urlparse, parse_qs, urlunparse
 from BeautifulSoup import BeautifulSoup
 from mongoengine.queryset import NotUniqueError
 from requests.exceptions import Timeout
@@ -99,10 +100,19 @@ class FeedGetter(object):
 
             real_url = article.link
             try:
+                # remove utm_* parameters
                 r = requests.get(article.link, timeout=2.0)
-                #parsed_url = urlparse(r.url)
-                #real_url = parsed_url.scheme + '://' + parsed_url.netloc + parsed_url.path
-                real_url = r.url
+                parsed_url = urlparse(r.url)
+                qd = parse_qs(parsed.query, keep_blank_values=True)
+                filtered = dict( (k, v) for k, v in qd.iteritems() if not k.startswith('utm_'))
+                real_url = urlunparse([
+                    parsed.scheme,
+                    parsed.netloc,
+                    parsed.path,
+                    parsed.params,
+                    urlencode(filtered, doseq=True), # query string
+                    parsed.fragment
+                ])
             except Timeout:
                 pyaggr3g470r_log.warning("Timeout when getting the real URL of %s." % (article.link,))
             except Exception as e:
-- 
cgit