aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCédric Bonhomme <kimble.mandel@gmail.com>2014-04-23 14:09:10 +0200
committerCédric Bonhomme <kimble.mandel@gmail.com>2014-04-23 14:09:10 +0200
commitf75bd130184f4a6d5a75279d6c5c0f8af9942da6 (patch)
tree990043a0d5b12b7904cba14546f64d032b5b9fc8
parentAutoindexation of new articles (not on Heroku). (diff)
downloadnewspipe-f75bd130184f4a6d5a75279d6c5c0f8af9942da6.tar.gz
newspipe-f75bd130184f4a6d5a75279d6c5c0f8af9942da6.tar.bz2
newspipe-f75bd130184f4a6d5a75279d6c5c0f8af9942da6.zip
Separate indexes by users.
-rw-r--r--pyaggr3g470r/crawler.py2
-rw-r--r--pyaggr3g470r/search.py23
-rw-r--r--pyaggr3g470r/views.py4
3 files changed, 16 insertions, 13 deletions
diff --git a/pyaggr3g470r/crawler.py b/pyaggr3g470r/crawler.py
index 661eddf0..dade3bea 100644
--- a/pyaggr3g470r/crawler.py
+++ b/pyaggr3g470r/crawler.py
@@ -244,7 +244,7 @@ class FeedGetter(object):
for element in articles:
article = Article.query.filter(Article.user_id == self.user.id, Article.link == element.link).first()
try:
- fastsearch.add_to_index([article], article.source)
+ fastsearch.add_to_index(self.user.id, [article], article.source)
except:
pyaggr3g470r_log.error("Problem during indexation.")
return True \ No newline at end of file
diff --git a/pyaggr3g470r/search.py b/pyaggr3g470r/search.py
index 9000b477..6f8168db 100644
--- a/pyaggr3g470r/search.py
+++ b/pyaggr3g470r/search.py
@@ -44,9 +44,10 @@ indexdir = "./pyaggr3g470r/var/indexdir"
schema = Schema(title=TEXT, \
content=TEXT, \
article_id=NUMERIC(int, stored=True), \
- feed_id=NUMERIC(int, stored=True))
+ feed_id=NUMERIC(int, stored=True), \
+ user_id=NUMERIC(int, stored=True))
-def create_index(feeds):
+def create_index(user):
"""
Creates the index.
"""
@@ -54,15 +55,16 @@ def create_index(feeds):
os.makedirs(indexdir)
ix = create_in(indexdir, schema)
writer = ix.writer()
- for feed in feeds:
+ for feed in user.feeds:
for article in feed.articles:
writer.add_document(title=article.title, \
content=utils.clear_string(article.content), \
article_id=article.id, \
- feed_id=feed.id)
+ feed_id=feed.id, \
+ user_id=user.id)
writer.commit()
-def add_to_index(articles, feed):
+def add_to_index(user_id, articles, feed):
"""
Add a list of articles to the index.
Here an AsyncWriter is used because the function will
@@ -79,10 +81,11 @@ def add_to_index(articles, feed):
writer.add_document(title=article.title, \
content=utils.clear_string(article.content), \
article_id=article.id, \
- feed_id=feed.id)
+ feed_id=feed.id, \
+ user_id=user_id)
writer.commit()
-def delete_article(feed_id, article_id):
+def delete_article(user_id, feed_id, article_id):
"""
Delete an article from the index.
"""
@@ -91,11 +94,11 @@ def delete_article(feed_id, article_id):
except (EmptyIndexError, OSError) as e:
raise EmptyIndexError
writer = ix.writer()
- document = And([Term("feed_id", feed_id), Term("article_id", article_id)])
+ document = And([Term("user_id", user_id), Term("feed_id", feed_id), Term("article_id", article_id)])
writer.delete_by_query(document)
writer.commit()
-def search(term):
+def search(user_id, term):
"""
Search for `term` in the index.
Returns a list of articles.
@@ -108,7 +111,7 @@ def search(term):
with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(term)
results = searcher.search(query, limit=None)
- for article in results:
+ for article in [item for item in results if item["user_id"] == user_id]:
result_dict[article["feed_id"]].append(article["article_id"])
return result_dict, len(results)
diff --git a/pyaggr3g470r/views.py b/pyaggr3g470r/views.py
index 8ec3978f..f3f4636f 100644
--- a/pyaggr3g470r/views.py
+++ b/pyaggr3g470r/views.py
@@ -410,7 +410,7 @@ def index_database():
"""
if not conf.ON_HEROKU:
user = User.query.filter(User.id == g.user.id).first()
- fastsearch.create_index(user.feeds)
+ fastsearch.create_index(user)
flash('Database indexed.', 'success')
return redirect(url_for('home'))
else:
@@ -459,7 +459,7 @@ def search():
result = []
query = request.args.get('query', None)
if query != None:
- results, nb_articles = fastsearch.search(query)
+ results, nb_articles = fastsearch.search(user.id, query)
for feed_id in results:
for feed in user.feeds:
if feed.id == feed_id:
bgstack15