aboutsummaryrefslogtreecommitdiff
path: root/pyaggr3g470r/feedgetter.py
diff options
context:
space:
mode:
Diffstat (limited to 'pyaggr3g470r/feedgetter.py')
-rw-r--r--pyaggr3g470r/feedgetter.py44
1 files changed, 24 insertions, 20 deletions
diff --git a/pyaggr3g470r/feedgetter.py b/pyaggr3g470r/feedgetter.py
index 2325a26e..1c834b88 100644
--- a/pyaggr3g470r/feedgetter.py
+++ b/pyaggr3g470r/feedgetter.py
@@ -34,9 +34,10 @@ from datetime import datetime
from urllib import urlencode
from urlparse import urlparse, parse_qs, urlunparse
from BeautifulSoup import BeautifulSoup
-from mongoengine.queryset import NotUniqueError
+
from requests.exceptions import Timeout
+from sqlalchemy.exc import IntegrityError
import models
import conf
@@ -44,7 +45,8 @@ import search
import utils
from flask.ext.mail import Message
-from pyaggr3g470r import app, mail
+from pyaggr3g470r import app, db, mail
+from pyaggr3g470r.models import User, Feed, Article
import log
pyaggr3g470r_log = log.Log("feedgetter")
@@ -73,7 +75,7 @@ class FeedGetter(object):
"https": "http://" + conf.HTTP_PROXY
}
feedparser.USER_AGENT = conf.USER_AGENT
- self.user = models.User.objects(email=email).first()
+ self.user = User.query.filter(User.email == email).first()
def retrieve_feed(self, feed_id=None):
"""
@@ -81,7 +83,7 @@ class FeedGetter(object):
"""
feeds = [feed for feed in self.user.feeds if feed.enabled]
if feed_id != None:
- feeds = [feed for feed in feeds if str(feed.oid) == feed_id]
+ feeds = [feed for feed in feeds if str(feed.id) == feed_id]
for current_feed in feeds:
try:
# launch a new thread for the RSS feed
@@ -96,8 +98,6 @@ class FeedGetter(object):
for th in list_of_threads:
th.join()
- self.user.save()
-
def process(self, feed):
"""
Retrieves articles form the feed and add them to the database.
@@ -170,24 +170,17 @@ class FeedGetter(object):
post_date = datetime(*article.updated_parsed[:6])
# save the article
- article = models.Article(post_date, nice_url, article_title, description, False, False)
- try:
- article.save()
- articles.append(article)
- pyaggr3g470r_log.info("New article %s (%s) added." % (article_title, nice_url))
- except NotUniqueError:
- pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article_title, nice_url))
- continue
- except Exception as e:
- pyaggr3g470r_log.error("Error when inserting article in database: " + str(e))
- continue
+ article = Article(link=nice_url, title=article_title,
+ content=description, readed=False, like=False, date=post_date)
+ articles.append(article)
# add the article to the Whoosh index
+ """
try:
search.add_to_index([article], feed)
except Exception as e:
pyaggr3g470r_log.error("Whoosh error.")
- pass
+ pass"""
# email notification
if conf.MAIL_ENABLED and feed.email_notification:
@@ -199,8 +192,19 @@ class FeedGetter(object):
mail.send(msg)
# add the articles to the list of articles for the current feed
- feed.articles.extend(articles)
- feed.articles = sorted(feed.articles, key=lambda t: t.date, reverse=True)
+ for article in articles:
+ try:
+ feed.articles.append(article)
+ db.session.merge(article)
+ db.session.commit()
+ pyaggr3g470r_log.info("New article %s (%s) added." % (article_title, nice_url))
+ except IntegrityError:
+ pyaggr3g470r_log.error("Article %s (%s) already in the database." % (article_title, nice_url))
+ db.session.rollback()
+ continue
+ except Exception as e:
+ pyaggr3g470r_log.error("Error when inserting article in database: " + str(e))
+ continue
return True
bgstack15