diff options
| -rw-r--r-- | conf.py | 5 | ||||
| -rw-r--r-- | conf/conf.cfg-sample | 18 | ||||
| -rw-r--r-- | pyaggr3g470r/__init__.py | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/decorators.py | 10 | ||||
| -rw-r--r-- | pyaggr3g470r/feedgetter.py | 36 | ||||
| -rwxr-xr-x | pyaggr3g470r/utils.py | 45 | ||||
| -rw-r--r-- | pyaggr3g470r/views.py | 3 |
7 files changed, 53 insertions, 71 deletions
@@ -27,7 +27,12 @@ WEBSERVER_DEBUG = int(config.get('webserver', 'debug')) == 1 WEBSERVER_HOST = config.get('webserver', 'host') WEBSERVER_PORT = int(config.get('webserver', 'port')) +MAIL_ENABLED = int(config.get('mail', 'enabled')) == 1 MAIL_HOST = config.get('mail', 'host') MAIL_PORT = int(config.get('mail', 'port')) +MAIL_TLS = int(config.get('mail', 'tls')) == 1 MAIL_SSL = int(config.get('mail', 'ssl')) == 1 MAIL_USERNAME = config.get('mail', 'username') +MAIL_PASSWORD = config.get('mail', 'password') +MAIL_FROM = config.get('mail', 'mail_from') +MAIL_TO = config.get('mail', 'mail_to') diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index 8e4953f7..a830996e 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -4,17 +4,17 @@ address = 127.0.0.1 port = 27017 username = root password = root -[crawler] -user_agent = Mozilla/5.0 (X11; Debian; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0 -thread_time_out = 10.0 -get_time_out = 5.0 [webserver] debug = 1 host = 0.0.0.0 port = 5000 [mail] -host = -port = 25 -ssl = 0 -username = -password = +enabled = 1 +host = smtp.googlemail.com +port = 465 +tls = 0 +ssl = 1 +username = your-gmail-username +password = your-gmail-password +mail_from = pyAggr3g470r@no-reply.com +mail_to = recipent-email diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index b24f3c7d..16bdc57f 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -19,15 +19,16 @@ app.config['SECRET_KEY'] = os.urandom(12) app.config['MONGODB_SETTINGS'] = {'DB': conf.DATABASE_NAME} app.config["MAIL_SERVER"] = conf.MAIL_HOST app.config["MAIL_PORT"] = conf.MAIL_PORT +app.config["MAIL_USE_TLS"] = conf.MAIL_TLS app.config["MAIL_USE_SSL"] = conf.MAIL_SSL app.config["MAIL_USERNAME"] = conf.MAIL_USERNAME -#app.config["MAIL_PASSWORD"] = 'your-password' +app.config["MAIL_PASSWORD"] = conf.MAIL_PASSWORD # Initializes the database db = MongoEngine(app) db.init_app(app) -from views import mail -mail.init_app(app) +from flask.ext.mail import Message, Mail +mail = Mail(app) from pyaggr3g470r import views
\ No newline at end of file diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py new file mode 100644 index 00000000..f6796224 --- /dev/null +++ b/pyaggr3g470r/decorators.py @@ -0,0 +1,10 @@ +#! /usr/bin/env python +#-*- coding: utf-8 -*- + +from threading import Thread + +def async(f): + def wrapper(*args, **kwargs): + thr = Thread(target = f, args = args, kwargs = kwargs) + thr.start() + return wrapper diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py index 421ce893..850f4449 100644 --- a/pyaggr3g470r/feedgetter.py +++ b/pyaggr3g470r/feedgetter.py @@ -39,6 +39,9 @@ import conf import search import utils +from flask.ext.mail import Message +from pyaggr3g470r import app, mail + #import log #pyaggr3g470r_log = log.Log() @@ -89,6 +92,7 @@ class FeedGetter(object): for article in a_feed['entries']: if models.Article.objects(link=article.link).first() != None: + # if article already in the database continue with the next article continue description = "" @@ -119,21 +123,23 @@ class FeedGetter(object): articles.append(article) """ - if self.articles.get_articles(feed_id, article_id) == []: - # add the article to the Whoosh index - try: - search.add_to_index([article], feed) - except: - print("Whoosh error.") - #pyaggr3g470r_log.error("Whoosh error.") - continue - - if conf.MAIL_ENABLED and feed["mail"]: - # if subscribed to the feed - threading.Thread(None, utils.send_mail, None, (conf.mail_from, conf.mail_to, \ - a_feed.feed.title, \ - article_title, description)).start() - """ + # add the article to the Whoosh index + try: + search.add_to_index([article], feed) + except: + print("Whoosh error.") + #pyaggr3g470r_log.error("Whoosh error.") + continue""" + + if conf.MAIL_ENABLED and feed.email_notification: + # if subscribed to the feed + with app.app_context(): + msg = Message('[pyAggr3g470r] ' + feed.title + ' : ' + article.title, \ + sender = conf.MAIL_FROM, recipients = [conf.MAIL_TO]) + msg.body = utils.clear_string(description) + msg.html = description + mail.send(msg) + feed.articles.extend(articles) feed.articles = sorted(feed.articles, key=lambda t: t.date, reverse=True) #feed.save() diff --git a/pyaggr3g470r/utils.py b/pyaggr3g470r/utils.py index f2108703..10614b05 100755 --- a/pyaggr3g470r/utils.py +++ b/pyaggr3g470r/utils.py @@ -40,10 +40,6 @@ import glob import operator import calendar -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - from BeautifulSoup import BeautifulSoup from collections import Counter @@ -109,9 +105,9 @@ def clear_string(data): Clear a string by removing HTML tags, HTML special caracters and consecutive white spaces (more that one). """ - p = re.compile(b'<[^>]+>') # HTML tags - q = re.compile(b'\s') # consecutive white spaces - return p.sub(b'', q.sub(b' ', bytes(data, "utf-8"))).decode("utf-8", "strict") + p = re.compile('<[^>]+>') # HTML tags + q = re.compile('\s') # consecutive white spaces + return p.sub('', q.sub(' ', data)) def normalize_filename(name): """ @@ -171,41 +167,6 @@ def tag_cloud(tags, query="word_count"): (min(1 + count * 7 / max([tag[1] for tag in tags]), 7), query, word, format(count, ',d'), calendar.month_name[int(word)])) \ for (word, count) in tags]) -def send_mail(mfrom, mto, feed_title, article_title, description): - """ - Send the article via mail. - """ - # Create the body of the message (a plain-text and an HTML version). - html = """<html>\n<head>\n<title>%s</title>\n</head>\n<body>\n%s\n</body>\n</html>""" % \ - (feed_title + ": " + article_title, description) - text = clear_string(description) - - # Create message container - the correct MIME type is multipart/alternative. - msg = MIMEMultipart('alternative') - msg['Subject'] = '[pyAggr3g470r] ' + feed_title + ": " + article_title - msg['From'] = mfrom - msg['To'] = mto - - # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(text, 'plain', 'utf-8') - part2 = MIMEText(html, 'html', 'utf-8') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg.attach(part1) - msg.attach(part2) - - # Send the message via local SMTP server. - try: - s = smtplib.SMTP(conf.smtp_server) - s.login(conf.username, conf.password) - except Exception as e: - print(e) - else: - s.send_message(msg) - s.quit() - def search_feed(url): """ Search a feed in a HTML page. diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views.py index 96d34aea..030272b9 100644 --- a/pyaggr3g470r/views.py +++ b/pyaggr3g470r/views.py @@ -29,7 +29,7 @@ __license__ = "GPLv3" from flask import render_template, request, flash, session, url_for, redirect, g from wtforms import TextField, PasswordField, SubmitField, validators -from flask.ext.mail import Message, Mail + from flask.ext.login import LoginManager, login_user, logout_user, login_required, current_user, AnonymousUserMixin from collections import defaultdict @@ -41,7 +41,6 @@ from pyaggr3g470r import app, db import feedgetter import models -mail = Mail() login_manager = LoginManager() login_manager.init_app(app) |