diff options
| -rw-r--r-- | conf.py | 14 | ||||
| -rw-r--r-- | conf/conf.cfg-sample | 7 | ||||
| -rw-r--r-- | pyaggr3g470r/__init__.py | 11 | ||||
| -rw-r--r-- | pyaggr3g470r/crawler.py | 51 | ||||
| -rw-r--r-- | pyaggr3g470r/decorators.py | 4 | ||||
| -rw-r--r-- | requirements.txt | 1 |
6 files changed, 51 insertions, 37 deletions
@@ -22,7 +22,8 @@ if not ON_HEROKU: config = confparser.SafeConfigParser() config.read("./conf/conf.cfg") - # Whoosh does not work on Heroku + PLATFORM_URL = config.get('misc', 'platform_url') + WHOOSH_ENABLED = True SQLALCHEMY_DATABASE_URI = config.get('database', 'uri') @@ -35,6 +36,7 @@ if not ON_HEROKU: WEBSERVER_HOST = config.get('webserver', 'host') WEBSERVER_PORT = int(config.get('webserver', 'port')) + ADMIN_EMAIL = config.get('mail', 'admin_email') MAIL_ENABLED = int(config.get('mail', 'enabled')) == 1 MAIL_HOST = config.get('mail', 'host') MAIL_PORT = int(config.get('mail', 'port')) @@ -42,24 +44,24 @@ if not ON_HEROKU: MAIL_SSL = int(config.get('mail', 'ssl')) == 1 MAIL_USERNAME = config.get('mail', 'username') MAIL_PASSWORD = config.get('mail', 'password') - MAIL_FROM = config.get('mail', 'mail_from') - MAIL_TO = config.get('mail', 'mail_to') WEBZINE_ROOT = PATH + "/pyaggr3g470r/var/export/" else: + PLATFORM_URL = os.environ.get('PLATFORM_URL', 'https://pyaggr3g470r.herokuapp.com/') + + SQLALCHEMY_DATABASE_URI = os.environ['DATABASE_URL'] + HTTP_PROXY = "" USER_AGENT = "Mozilla/5.0 (X11; Debian; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0" RESOLVE_ARTICLE_URL = int(os.environ.get('RESOLVE_ARTICLE_URL', 0)) == 1 WEBSERVER_DEBUG = False - WEBSERVER_HOST ='0.0.0.0' + WEBSERVER_HOST = '0.0.0.0' WEBSERVER_PORT = int(os.environ.get('PORT', 5000)) MAIL_ENABLED = False - SQLALCHEMY_DATABASE_URI = os.environ['DATABASE_URL'] - WEBZINE_ROOT = "/tmp/" diff --git a/conf/conf.cfg-sample b/conf/conf.cfg-sample index e4a44417..5af7d0af 100644 --- a/conf/conf.cfg-sample +++ b/conf/conf.cfg-sample @@ -1,3 +1,5 @@ +[misc] +platform_url = https://pyaggr3g470r.herokuapp.com/ [database] uri = postgres://cedric:password@127.0.0.1:5432/pyAggr3g470r [feedparser] @@ -9,12 +11,11 @@ debug = 1 host = 0.0.0.0 port = 5000 [mail] -enabled = 1 +enabled = 0 +admin_email = pyAggr3g470r@no-reply.com host = smtp.googlemail.com port = 465 tls = 0 ssl = 1 username = your-gmail-username password = your-gmail-password -mail_from = pyAggr3g470r@no-reply.com -mail_to = recipent-email diff --git a/pyaggr3g470r/__init__.py b/pyaggr3g470r/__init__.py index a8211217..46498679 100644 --- a/pyaggr3g470r/__init__.py +++ b/pyaggr3g470r/__init__.py @@ -27,17 +27,6 @@ def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS -if not conf.ON_HEROKU: - app.config["MAIL_SERVER"] = conf.MAIL_HOST - app.config["MAIL_PORT"] = conf.MAIL_PORT - app.config["MAIL_USE_TLS"] = conf.MAIL_TLS - app.config["MAIL_USE_SSL"] = conf.MAIL_SSL - app.config["MAIL_USERNAME"] = conf.MAIL_USERNAME - app.config["MAIL_PASSWORD"] = conf.MAIL_PASSWORD - - from flask.ext.mail import Mail - mail = Mail(app) - # Gravatar gravatar = Gravatar(app, size=100, rating='g', default='retro', force_default=False, use_ssl=False, base_url=None) diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py index 955d2144..34ce9d74 100644 --- a/pyaggr3g470r/crawler.py +++ b/pyaggr3g470r/crawler.py @@ -40,12 +40,11 @@ from gevent.pool import Pool import log import utils import conf +import emails from pyaggr3g470r import db from pyaggr3g470r.models import User, Article if not conf.ON_HEROKU: import search as fastsearch - from flask.ext.mail import Message - from pyaggr3g470r import mail pyaggr3g470r_log = log.Log("feedgetter") @@ -100,11 +99,15 @@ class FeedGetter(object): elements = [item.value for item in responses if item.value is not None] # 3 - Insert articles in the database - self.insert_database(elements) + new_articles = self.insert_database(elements) # 4 - Indexation if not conf.ON_HEROKU: - self.index(elements) + self.index(new_articles) + + # 5 - Mail notification + if not conf.ON_HEROKU and conf.MAIL_ENABLED: + self.mail_notification(new_articles) pyaggr3g470r_log.info("All articles retrieved. End of the processus.") @@ -203,9 +206,11 @@ class FeedGetter(object): Insert articles in the database. """ pyaggr3g470r_log.info("Database insertion...") + new_articles = [] for feed, articles in elements: for article in articles: + exist = Article.query.filter(Article.user_id == self.user.id, Article.feed_id == feed.id, @@ -214,6 +219,7 @@ class FeedGetter(object): pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article.title, article.link)) continue + new_articles.append(article) try: feed.articles.append(article) @@ -223,26 +229,39 @@ class FeedGetter(object): except IntegrityError: pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article.title, article.link)) + articles.remove(article) db.session.rollback() continue except Exception as e: pyaggr3g470r_log.error("Error when inserting article in database: " + str(e)) continue #db.session.close() - return True + return new_articles - def index(self, elements): + def index(self, new_articles): """ Index new articles. """ pyaggr3g470r_log.info("Indexing new articles.") - for feed, articles in elements: - for element in articles: - article = Article.query.filter(Article.user_id == self.user.id, - Article.link == element.link).first() - try: - fastsearch.add_to_index(self.user.id, [article], - article.source) - except: - pyaggr3g470r_log.error("Problem during indexation.") - return True
\ No newline at end of file + for element in new_articles: + article = Article.query.filter(Article.user_id == self.user.id, + Article.link == element.link).first() + try: + fastsearch.add_to_index(self.user.id, [article], + article.source) + except: + pyaggr3g470r_log.error("Problem during indexation.") + return True + + def mail_notification(self, new_articles): + """ + Mail notification. + """ + pyaggr3g470r_log.info("Starting mail notification.") + for element in new_articles: + if element.source.email_notification: + emails.new_article_notification(self.user, element.source, element) + + return True + +
\ No newline at end of file diff --git a/pyaggr3g470r/decorators.py b/pyaggr3g470r/decorators.py index 0052dc80..80568add 100644 --- a/pyaggr3g470r/decorators.py +++ b/pyaggr3g470r/decorators.py @@ -10,6 +10,10 @@ from pyaggr3g470r.models import Feed def async(f): + """ + This decorator enables to send email in a new thread. + This prevent the server to freeze. + """ def wrapper(*args, **kwargs): thr = Thread(target=f, args=args, kwargs=kwargs) thr.start() diff --git a/requirements.txt b/requirements.txt index f9b62eaa..5e2efdac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ Flask-SQLAlchemy Flask-Login Flask-Principal Flask-WTF -Flask-Mail Flask-Gravatar WTForms gevent |