aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r
diff options
context:
space:
mode:
Diffstat (limited to 'pyaggr3g470r')
-rw-r--r--pyaggr3g470r/__init__.py11
-rw-r--r--pyaggr3g470r/crawler.py51
-rw-r--r--pyaggr3g470r/decorators.py4
3 files changed, 39 insertions, 27 deletions
diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py
index a8211217..46498679 100644
--- a/pyaggr3g470r/__init__.py
+++ b/pyaggr3g470r/__init__.py
@@ -27,17 +27,6 @@ def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
-if not conf.ON_HEROKU:
- app.config["MAIL_SERVER"] = conf.MAIL_HOST
- app.config["MAIL_PORT"] = conf.MAIL_PORT
- app.config["MAIL_USE_TLS"] = conf.MAIL_TLS
- app.config["MAIL_USE_SSL"] = conf.MAIL_SSL
- app.config["MAIL_USERNAME"] = conf.MAIL_USERNAME
- app.config["MAIL_PASSWORD"] = conf.MAIL_PASSWORD
-
- from flask.ext.mail import Mail
- mail = Mail(app)
-
# Gravatar
gravatar = Gravatar(app, size=100, rating='g', default='retro',
force_default=False, use_ssl=False, base_url=None)
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index 955d2144..34ce9d74 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -40,12 +40,11 @@ from gevent.pool import Pool
import log
import utils
import conf
+import emails
from pyaggr3g470r import db
from pyaggr3g470r.models import User, Article
if not conf.ON_HEROKU:
import search as fastsearch
- from flask.ext.mail import Message
- from pyaggr3g470r import mail
pyaggr3g470r_log = log.Log("feedgetter")
@@ -100,11 +99,15 @@ class FeedGetter(object):
elements = [item.value for item in responses if item.value is not None]
# 3 - Insert articles in the database
- self.insert_database(elements)
+ new_articles = self.insert_database(elements)
# 4 - Indexation
if not conf.ON_HEROKU:
- self.index(elements)
+ self.index(new_articles)
+
+ # 5 - Mail notification
+ if not conf.ON_HEROKU and conf.MAIL_ENABLED:
+ self.mail_notification(new_articles)
pyaggr3g470r_log.info("All articles retrieved. End of the processus.")
@@ -203,9 +206,11 @@ class FeedGetter(object):
Insert articles in the database.
"""
pyaggr3g470r_log.info("Database insertion...")
+ new_articles = []
for feed, articles in elements:
for article in articles:
+
exist = Article.query.filter(Article.user_id == self.user.id,
Article.feed_id == feed.id,
@@ -214,6 +219,7 @@ class FeedGetter(object):
pyaggr3g470r_log.error("Article %s (%s) already in the database." %
(article.title, article.link))
continue
+ new_articles.append(article)
try:
feed.articles.append(article)
@@ -223,26 +229,39 @@ class FeedGetter(object):
except IntegrityError:
pyaggr3g470r_log.error("Article %s (%s) already in the database." %
(article.title, article.link))
+ articles.remove(article)
db.session.rollback()
continue
except Exception as e:
pyaggr3g470r_log.error("Error when inserting article in database: " + str(e))
continue
#db.session.close()
- return True
+ return new_articles
- def index(self, elements):
+ def index(self, new_articles):
"""
Index new articles.
"""
pyaggr3g470r_log.info("Indexing new articles.")
- for feed, articles in elements:
- for element in articles:
- article = Article.query.filter(Article.user_id == self.user.id,
- Article.link == element.link).first()
- try:
- fastsearch.add_to_index(self.user.id, [article],
- article.source)
- except:
- pyaggr3g470r_log.error("Problem during indexation.")
- return True \ No newline at end of file
+ for element in new_articles:
+ article = Article.query.filter(Article.user_id == self.user.id,
+ Article.link == element.link).first()
+ try:
+ fastsearch.add_to_index(self.user.id, [article],
+ article.source)
+ except:
+ pyaggr3g470r_log.error("Problem during indexation.")
+ return True
+
+ def mail_notification(self, new_articles):
+ """
+ Mail notification.
+ """
+ pyaggr3g470r_log.info("Starting mail notification.")
+ for element in new_articles:
+ if element.source.email_notification:
+ emails.new_article_notification(self.user, element.source, element)
+
+ return True
+
+ \ No newline at end of file
diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py
index 0052dc80..80568add 100644
--- a/pyaggr3g470r/decorators.py
+++ b/pyaggr3g470r/decorators.py
@@ -10,6 +10,10 @@ from pyaggr3g470r.models import Feed
def async(f):
+ """
+ This decorator enables to send email in a new thread.
+ This prevent the server to freeze.
+ """
def wrapper(*args, **kwargs):
thr = Thread(target=f, args=args, kwargs=kwargs)
thr.start()
bgstack15