aboutsummaryrefslogtreecommitdiff
path: root/benchmark
diff options
context:
space:
mode:
Diffstat (limited to 'benchmark')
-rw-r--r--benchmark/__init__.py1
-rw-r--r--benchmark/testelasticsearch.py58
-rw-r--r--benchmark/testwhoosh.py50
3 files changed, 109 insertions, 0 deletions
diff --git a/benchmark/__init__.py b/benchmark/__init__.py
new file mode 100644
index 00000000..8d1c8b69
--- /dev/null
+++ b/benchmark/__init__.py
@@ -0,0 +1 @@
+
diff --git a/benchmark/testelasticsearch.py b/benchmark/testelasticsearch.py
new file mode 100644
index 00000000..7a5d1d5b
--- /dev/null
+++ b/benchmark/testelasticsearch.py
@@ -0,0 +1,58 @@
+#! /usr/bin/env python
+#-*- coding: utf-8 -*-
+
+import elasticsearch
+from elasticsearch import client
+
+from pyaggr3g470r import utils
+
+# Connect to Elasticsearch node specified in the configuration file:
+es = elasticsearch.Elasticsearch(hosts={"127.0.0.1" : 9200})
+
+def delete_index():
+ """
+ Deletes all indexes.
+ """
+ es = elasticsearch.Elasticsearch(hosts={"127.0.0.1" : 9200})
+ ic = client.IndicesClient(es.indices.client)
+ try:
+ ic.delete("")
+ except:
+ pass
+
+def create_index(articles):
+ """
+ Creates the index.
+ """
+ for article in articles:
+ res = es.index(
+ index="pyaggr3g470r",
+ doc_type="text",
+ id=str(article.id),
+ body={
+ "title": article.title,
+ "content": utils.clear_string(article.content)
+ }
+ )
+ return True
+
+def search(term):
+ """
+ Search a term.
+ """
+ try:
+ return es.search(index="pyaggr3g470r", body=
+ {"query" : {
+ "filtered" : {
+ "query" : {
+ "query_string" : {
+ "default_field" : "content",
+ "query" : term
+ }
+ }
+ }
+ }
+ }, size=5000)
+ except elasticsearch.exceptions.NotFoundError as e:
+ logger.warning(str(e))
+ return None \ No newline at end of file
diff --git a/benchmark/testwhoosh.py b/benchmark/testwhoosh.py
new file mode 100644
index 00000000..b488dcd6
--- /dev/null
+++ b/benchmark/testwhoosh.py
@@ -0,0 +1,50 @@
+#! /usr/bin/env python
+#-*- coding: utf-8 -*-
+
+
+import os
+
+from whoosh.index import create_in, open_dir
+from whoosh.index import EmptyIndexError
+from whoosh.fields import *
+from whoosh.query import *
+from whoosh.qparser import QueryParser
+
+from pyaggr3g470r import utils
+
+indexdir = "./pyaggr3g470r/var/indexdir"
+
+schema = Schema(title=TEXT(stored=True), content=TEXT)
+
+def create_index(articles):
+ """
+ Creates the index.
+ """
+ ix = create_in(indexdir, schema)
+ writer = ix.writer()
+ for article in articles:
+ writer.add_document(content=utils.clear_string(article.content))
+ writer.commit()
+
+def search(term):
+ """
+ Search for `term` in the index.
+ Returns a list of articles.
+ """
+ try:
+ ix = open_dir(indexdir)
+ except (EmptyIndexError, OSError) as e:
+ raise EmptyIndexError
+ with ix.searcher() as searcher:
+ query = QueryParser("content", ix.schema).parse(term)
+ results = searcher.search(query, limit=None)
+ #return [(article["feed_id"], article["article_id"]) for article in results]
+
+
+if __name__ == "__main__":
+ # Point of entry in execution mode.
+ #create_index()
+ print(nb_documents())
+ results = search("Nothomb")
+ for article in results:
+ print(article)
bgstack15