From b2618e9404b84cc62d4becb02436233a0d53b375 Mon Sep 17 00:00:00 2001 From: Cédric Bonhomme Date: Wed, 25 Nov 2015 22:45:43 +0100 Subject: Rfactorization. Just a start... --- web/lib/utils.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 web/lib/utils.py (limited to 'web/lib/utils.py') diff --git a/web/lib/utils.py b/web/lib/utils.py new file mode 100644 index 00000000..aa552a12 --- /dev/null +++ b/web/lib/utils.py @@ -0,0 +1,57 @@ +import types +import urllib +import logging +import requests +from hashlib import md5 + +logger = logging.getLogger(__name__) + + +def default_handler(obj): + """JSON handler for default query formatting""" + if hasattr(obj, 'isoformat'): + return obj.isoformat() + if hasattr(obj, 'dump'): + return obj.dump() + if isinstance(obj, (set, frozenset, types.GeneratorType)): + return list(obj) + if isinstance(obj, BaseException): + return str(obj) + raise TypeError("Object of type %s with value of %r " + "is not JSON serializable" % (type(obj), obj)) + + +def try_keys(dico, *keys): + for key in keys: + if key in dico: + return dico[key] + return + + +def rebuild_url(url, base_split): + split = urllib.parse.urlsplit(url) + if split.scheme and split.netloc: + return url # url is fine + new_split = urllib.parse.SplitResult( + scheme=split.scheme or base_split.scheme, + netloc=split.netloc or base_split.netloc, + path=split.path, query='', fragment='') + return urllib.parse.urlunsplit(new_split) + + +def try_get_icon_url(url, *splits): + for split in splits: + if split is None: + continue + rb_url = rebuild_url(url, split) + response = requests.get(rb_url, verify=False, timeout=10) + # if html in content-type, we assume it's a fancy 404 page + content_type = response.headers.get('content-type', '') + if response.ok and 'html' not in content_type and response.content: + return response.url + return None + + +def to_hash(text): + return md5(text.encode('utf8') if hasattr(text, 'encode') else text)\ + .hexdigest() -- cgit