aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel@gmail.com>2013-07-24 14:45:48 +0200
committerCédric Bonhomme <kimble.mandel@gmail.com>2013-07-24 14:45:48 +0200
commit86b78a97e384475504ce6845846e547f70c47693 (patch)
treea673dca86cbe8d99abd8fa054d0c1089542958c6
parentadded add_to_index function, to add an article to the Whoosh index. (diff)
downloadnewspipe-86b78a97e384475504ce6845846e547f70c47693.tar.gz
newspipe-86b78a97e384475504ce6845846e547f70c47693.tar.bz2
newspipe-86b78a97e384475504ce6845846e547f70c47693.zip
AsyncWriter is used to prevent whoosh.store.LockError error. And we only try to add new articles to the index.
-rwxr-xr-xsource/feedgetter.py11
-rw-r--r--source/search.py5
2 files changed, 10 insertions, 6 deletions
diff --git a/source/feedgetter.py b/source/feedgetter.py
index bd9cacc9..a9f0b01c 100755
--- a/source/feedgetter.py
+++ b/source/feedgetter.py
@@ -178,12 +178,13 @@ class FeedGetter(object):
articles.append(article)
- # add the article to the Whoosh index
- #search.add_to_index([article], feed)
+ if self.articles.get_articles(feed_id, article_id) == False:
+ # add the article to the Whoosh index
+ search.add_to_index([article], feed)
- if conf.MAIL_ENABLED and feed["mail"] and self.articles.get_articles(feed_id, article_id) == False:
- # if subscribed to the feed AND if article not already in the database
- threading.Thread(None, utils.send_mail, None, (conf.mail_from, conf.mail_to, \
+ if conf.MAIL_ENABLED and feed["mail"]:
+ # if subscribed to the feed
+ threading.Thread(None, utils.send_mail, None, (conf.mail_from, conf.mail_to, \
a_feed.feed.title, \
article_title, description)).start()
self.articles.add_articles(articles, feed_id)
diff --git a/source/search.py b/source/search.py
index 63abddc4..f38133eb 100644
--- a/source/search.py
+++ b/source/search.py
@@ -32,6 +32,7 @@ from whoosh.index import create_in, open_dir
from whoosh.index import EmptyIndexError
from whoosh.fields import *
from whoosh.qparser import QueryParser
+from whoosh.writing import AsyncWriter
import conf
import utils
@@ -66,12 +67,14 @@ def create_index():
def add_to_index(articles, feed):
"""
Add a list of articles to the index.
+ Here an AsyncWriter is used because the function will
+ be called in multiple threads by the feedgetter module.
"""
try:
ix = open_dir(indexdir)
except (EmptyIndexError, OSError) as e:
raise EmptyIndexError
- writer = ix.writer()
+ writer = AsyncWriter(ix)
for article in articles:
writer.add_document(title=article["article_title"], \
content=utils.clear_string(article["article_content"]), \
bgstack15