diff options
Diffstat (limited to 'pyaggr3g470r')
| -rw-r--r-- | pyaggr3g470r/__init__.py | 11 | ||||
| -rw-r--r-- | pyaggr3g470r/crawler.py | 51 | ||||
| -rw-r--r-- | pyaggr3g470r/decorators.py | 4 |
3 files changed, 39 insertions, 27 deletions
diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index a8211217..46498679 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -27,17 +27,6 @@ def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS -if not conf.ON_HEROKU: - app.config["MAIL_SERVER"] = conf.MAIL_HOST - app.config["MAIL_PORT"] = conf.MAIL_PORT - app.config["MAIL_USE_TLS"] = conf.MAIL_TLS - app.config["MAIL_USE_SSL"] = conf.MAIL_SSL - app.config["MAIL_USERNAME"] = conf.MAIL_USERNAME - app.config["MAIL_PASSWORD"] = conf.MAIL_PASSWORD - - from flask.ext.mail import Mail - mail = Mail(app) - # Gravatar gravatar = Gravatar(app, size=100, rating='g', default='retro', force_default=False, use_ssl=False, base_url=None) diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index 955d2144..34ce9d74 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -40,12 +40,11 @@ from gevent.pool import Pool import log import utils import conf +import emails from pyaggr3g470r import db from pyaggr3g470r.models import User, Article if not conf.ON_HEROKU: import search as fastsearch - from flask.ext.mail import Message - from pyaggr3g470r import mail pyaggr3g470r_log = log.Log("feedgetter") @@ -100,11 +99,15 @@ class FeedGetter(object): elements = [item.value for item in responses if item.value is not None] # 3 - Insert articles in the database - self.insert_database(elements) + new_articles = self.insert_database(elements) # 4 - Indexation if not conf.ON_HEROKU: - self.index(elements) + self.index(new_articles) + + # 5 - Mail notification + if not conf.ON_HEROKU and conf.MAIL_ENABLED: + self.mail_notification(new_articles) pyaggr3g470r_log.info("All articles retrieved. End of the processus.") @@ -203,9 +206,11 @@ class FeedGetter(object): Insert articles in the database. """ pyaggr3g470r_log.info("Database insertion...") + new_articles = [] for feed, articles in elements: for article in articles: + exist = Article.query.filter(Article.user_id == self.user.id, Article.feed_id == feed.id, @@ -214,6 +219,7 @@ class FeedGetter(object): pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article.title, article.link)) continue + new_articles.append(article) try: feed.articles.append(article) @@ -223,26 +229,39 @@ class FeedGetter(object): except IntegrityError: pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article.title, article.link)) + articles.remove(article) db.session.rollback() continue except Exception as e: pyaggr3g470r_log.error("Error when inserting article in database: " + str(e)) continue #db.session.close() - return True + return new_articles - def index(self, elements): + def index(self, new_articles): """ Index new articles. """ pyaggr3g470r_log.info("Indexing new articles.") - for feed, articles in elements: - for element in articles: - article = Article.query.filter(Article.user_id == self.user.id, - Article.link == element.link).first() - try: - fastsearch.add_to_index(self.user.id, [article], - article.source) - except: - pyaggr3g470r_log.error("Problem during indexation.") - return True
\ No newline at end of file + for element in new_articles: + article = Article.query.filter(Article.user_id == self.user.id, + Article.link == element.link).first() + try: + fastsearch.add_to_index(self.user.id, [article], + article.source) + except: + pyaggr3g470r_log.error("Problem during indexation.") + return True + + def mail_notification(self, new_articles): + """ + Mail notification. + """ + pyaggr3g470r_log.info("Starting mail notification.") + for element in new_articles: + if element.source.email_notification: + emails.new_article_notification(self.user, element.source, element) + + return True + +
\ No newline at end of file diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index 0052dc80..80568add 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -10,6 +10,10 @@ from pyaggr3g470r.models import Feed def async(f): + """ + This decorator enables to send email in a new thread. + This prevent the server to freeze. + """ def wrapper(*args, **kwargs): thr = Thread(target=f, args=args, kwargs=kwargs) thr.start() |