aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/utils.py
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel@gmail.com>2013-10-13 10:07:51 +0200
committerCédric Bonhomme <kimble.mandel@gmail.com>2013-10-13 10:07:51 +0200
commitabec11e7ca0ce49081343bb2b2eb8520058d67a8 (patch)
tree9aeb6f19650c4d62a4b539ad0c1a9d1f83c6c843 /pyaggr3g470r/utils.py
parentRemoved all .py files. (diff)
downloadnewspipe-abec11e7ca0ce49081343bb2b2eb8520058d67a8.tar.gz
newspipe-abec11e7ca0ce49081343bb2b2eb8520058d67a8.tar.bz2
newspipe-abec11e7ca0ce49081343bb2b2eb8520058d67a8.zip
Added new files. First prototype with the Flask micro-framework.
Diffstat (limited to 'pyaggr3g470r/utils.py')
-rwxr-xr-xpyaggr3g470r/utils.py255
1 files changed, 255 insertions, 0 deletions
diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py
new file mode 100755
index 00000000..5391604c
--- /dev/null
+++ b/pyaggr3g470r/utils.py
@@ -0,0 +1,255 @@
+#! /usr/bin/env python
+#-*- coding: utf-8 -*-
+
+# pyAggr3g470r - A Web based news aggregator.
+# Copyright (C) 2010-2013 Cédric Bonhomme - http://cedricbonhomme.org/
+#
+# For more information : http://bitbucket.org/cedricbonhomme/pyaggr3g470r/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>
+
+__author__ = "Cedric Bonhomme"
+__version__ = "$Revision: 1.5 $"
+__date__ = "$Date: 2010/12/07 $"
+__revision__ = "$Date: 2013/07/24 $"
+__copyright__ = "Copyright (c) Cedric Bonhomme"
+__license__ = "GPLv3"
+
+#
+# This file provides functions used for:
+# - the database management;
+# - generation of tags cloud;
+# - HTML processing;
+# - e-mail notifications.
+#
+
+import os
+import re
+import glob
+import operator
+import calendar
+
+try:
+ from qrcode.pyqrnative.PyQRNative import QRCode, QRErrorCorrectLevel, CodeOverflowException
+ from qrcode import qr
+except:
+ pass
+
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+import requests
+from BeautifulSoup import BeautifulSoup
+
+from collections import Counter
+from contextlib import contextmanager
+
+import conf
+
+# regular expression to check URL
+url_finders = [ \
+ re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?/[-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]*[^]'\\.}>\\),\\\"]"), \
+ re.compile("([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|(((news|telnet|nttp|file|http|ftp|https)://)|(www|ftp)[-A-Za-z0-9]*\\.)[-A-Za-z0-9\\.]+)(:[0-9]*)?"), \
+ re.compile("(~/|/|\\./)([-A-Za-z0-9_\\$\\.\\+\\!\\*\\(\\),;:@&=\\?/~\\#\\%]|\\\\)+"), \
+ re.compile("'\\<((mailto:)|)[-A-Za-z0-9\\.]+@[-A-Za-z0-9\\.]+") \
+]
+
+import log
+pyaggr3g470r_log = log.Log()
+
+@contextmanager
+def opened_w_error(filename, mode="r"):
+ try:
+ f = open(filename, mode)
+ except IOError as err:
+ yield None, err
+ else:
+ try:
+ yield f, None
+ finally:
+ f.close()
+
+def open_url(url):
+ """
+ Open an URL with the proxy and the user-agent
+ specified in the configuration file.
+ """
+ if conf.HTTP_PROXY == "":
+ proxy = {}
+ else:
+ proxy = {"http" : conf.HTTP_PROXY}
+ opener = urllib.request.FancyURLopener(proxy)
+ try:
+ opener = urllib.request.build_opener()
+ opener.addheaders = [('User-agent', conf.USER_AGENT)]
+ return (True, opener.open(url))
+ except urllib.error.HTTPError as e:
+ # server couldn't fulfill the request
+ error = (url, e.code, \
+ http.server.BaseHTTPRequestHandler.responses[e.code][1])
+ pyaggr3g470r_log.error(url + " " + str(e.code) + " " + \
+ http.server.BaseHTTPRequestHandler.responses[e.code][1])
+ return (False, error)
+ except urllib.error.URLError as e:
+ # failed to reach the server
+ if type(e.reason) == str:
+ error = (url, e.reason, e.reason)
+ pyaggr3g470r_log.error(url + " " + e.reason)
+ else:
+ error = (url, e.reason.errno, e.reason.strerror)
+ pyaggr3g470r_log.error(url + " " + str(e.reason.errno) + " " + \
+ e.reason.strerror)
+ return (False, error)
+
+def generate_qr_code(article):
+ """
+ Generated a QR Code for the article given in parameter.
+ """
+ try:
+ os.makedirs("./var/qrcode/")
+ except OSError:
+ pass
+ if not os.path.isfile("./var/qrcode/" + article["article_id"] + ".png"):
+ # QR Code generation
+ try:
+ f = qr.QRUrl(url = article["article_link"])
+ f.make()
+ f.save("./var/qrcode/" + article["article_id"] + ".png")
+ except:
+ pass
+
+def clear_string(data):
+ """
+ Clear a string by removing HTML tags, HTML special caracters
+ and consecutive white spaces (more that one).
+ """
+ p = re.compile(b'<[^>]+>') # HTML tags
+ q = re.compile(b'\s') # consecutive white spaces
+ return p.sub(b'', q.sub(b' ', bytes(data, "utf-8"))).decode("utf-8", "strict")
+
+def normalize_filename(name):
+ """
+ Normalize a file name.
+ """
+ file_name = re.sub("[,'!?|&]", "", name)
+ file_name = re.sub("[\s.]", "_", file_name)
+ file_name = file_name.strip('_')
+ file_name = file_name.strip('.')
+ return os.path.normpath(file_name)
+
+def load_stop_words():
+ """
+ Load the stop words and return them in a list.
+ """
+ stop_words_lists = glob.glob('./var/stop_words/*.txt')
+ stop_words = []
+
+ for stop_wods_list in stop_words_lists:
+ with opened_w_error(stop_wods_list, "r") as (stop_wods_file, err):
+ if err:
+ stop_words = []
+ else:
+ stop_words += stop_wods_file.read().split(";")
+ return stop_words
+
+def top_words(articles, n=10, size=5):
+ """
+ Return the n most frequent words in a list.
+ """
+ stop_words = load_stop_words()
+ words = Counter()
+ wordre = re.compile(r'\b\w{%s,}\b' % size, re.I)
+ for article in articles:
+ for word in [elem.lower() for elem in
+ wordre.findall(clear_string(article["article_content"])) \
+ if elem.lower() not in stop_words]:
+ words[word] += 1
+ return words.most_common(n)
+
+def tag_cloud(tags, query="word_count"):
+ """
+ Generates a tags cloud.
+ """
+ tags.sort(key=operator.itemgetter(0))
+ if query == "word_count":
+ # tags cloud from the management page
+ return ' '.join([('<font size=%d><a href="/search/?query=%s" title="Count: %s">%s</a></font>\n' % \
+ (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), word, format(count, ',d'), word)) \
+ for (word, count) in tags])
+ if query == "year":
+ # tags cloud for the history
+ return ' '.join([('<font size=%d><a href="/history/?query=%s:%s" title="Count: %s">%s</a></font>\n' % \
+ (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), query, word, format(count, ',d'), word)) \
+ for (word, count) in tags])
+ return ' '.join([('<font size=%d><a href="/history/?query=%s:%s" title="Count: %s">%s</a></font>\n' % \
+ (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), query, word, format(count, ',d'), calendar.month_name[int(word)])) \
+ for (word, count) in tags])
+
+def send_mail(mfrom, mto, feed_title, article_title, description):
+ """
+ Send the article via mail.
+ """
+ # Create the body of the message (a plain-text and an HTML version).
+ html = """<html>\n<head>\n<title>%s</title>\n</head>\n<body>\n%s\n</body>\n</html>""" % \
+ (feed_title + ": " + article_title, description)
+ text = clear_string(description)
+
+ # Create message container - the correct MIME type is multipart/alternative.
+ msg = MIMEMultipart('alternative')
+ msg['Subject'] = '[pyAggr3g470r] ' + feed_title + ": " + article_title
+ msg['From'] = mfrom
+ msg['To'] = mto
+
+ # Record the MIME types of both parts - text/plain and text/html.
+ part1 = MIMEText(text, 'plain', 'utf-8')
+ part2 = MIMEText(html, 'html', 'utf-8')
+
+ # Attach parts into message container.
+ # According to RFC 2046, the last part of a multipart message, in this case
+ # the HTML message, is best and preferred.
+ msg.attach(part1)
+ msg.attach(part2)
+
+ # Send the message via local SMTP server.
+ try:
+ s = smtplib.SMTP(conf.smtp_server)
+ s.login(conf.username, conf.password)
+ except Exception as e:
+ print(e)
+ else:
+ s.send_message(msg)
+ s.quit()
+
+def search_feed(url):
+ """
+ Search a feed in a HTML page.
+ """
+ soup, page = None, None
+ try:
+ result = open_url(url)
+ if result[0] == True:
+ page = open_url(url)[1]
+ else:
+ return None
+ soup = BeautifulSoup(page)
+ except:
+ return None
+ feed_links = soup('link', type='application/atom+xml')
+ feed_links.extend(soup('link', type='application/rss+xml'))
+ for feed_link in feed_links:
+ #if url not in feed_link['href']:
+ #return urllib.parse.urljoin(url, feed_link['href'])
+ return feed_link['href']
+ return None
bgstack15