diff options
Diffstat (limited to 'benchmark')
| -rw-r--r-- | benchmark/__init__.py | 1 | ||||
| -rw-r--r-- | benchmark/testelasticsearch.py | 58 | ||||
| -rw-r--r-- | benchmark/testwhoosh.py | 50 |
3 files changed, 109 insertions, 0 deletions
diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 00000000..8d1c8b69 --- /dev/null +++ b/benchmark/__init__.py @@ -0,0 +1 @@ + diff --git a/benchmark/testelasticsearch.py b/benchmark/testelasticsearch.py new file mode 100644 index 00000000..7a5d1d5b --- /dev/null +++ b/benchmark/testelasticsearch.py @@ -0,0 +1,58 @@ +#! /usr/bin/env python +#-*- coding: utf-8 -*- + +import elasticsearch +from elasticsearch import client + +from pyaggr3g470r import utils + +# Connect to Elasticsearch node specified in the configuration file: +es = elasticsearch.Elasticsearch(hosts={"127.0.0.1" : 9200}) + +def delete_index(): + """ + Deletes all indexes. + """ + es = elasticsearch.Elasticsearch(hosts={"127.0.0.1" : 9200}) + ic = client.IndicesClient(es.indices.client) + try: + ic.delete("") + except: + pass + +def create_index(articles): + """ + Creates the index. + """ + for article in articles: + res = es.index( + index="pyaggr3g470r", + doc_type="text", + id=str(article.id), + body={ + "title": article.title, + "content": utils.clear_string(article.content) + } + ) + return True + +def search(term): + """ + Search a term. + """ + try: + return es.search(index="pyaggr3g470r", body= + {"query" : { + "filtered" : { + "query" : { + "query_string" : { + "default_field" : "content", + "query" : term + } + } + } + } + }, size=5000) + except elasticsearch.exceptions.NotFoundError as e: + logger.warning(str(e)) + return None
\ No newline at end of file diff --git a/benchmark/testwhoosh.py b/benchmark/testwhoosh.py new file mode 100644 index 00000000..b488dcd6 --- /dev/null +++ b/benchmark/testwhoosh.py @@ -0,0 +1,50 @@ +#! /usr/bin/env python +#-*- coding: utf-8 -*- + + +import os + +from whoosh.index import create_in, open_dir +from whoosh.index import EmptyIndexError +from whoosh.fields import * +from whoosh.query import * +from whoosh.qparser import QueryParser + +from pyaggr3g470r import utils + +indexdir = "./pyaggr3g470r/var/indexdir" + +schema = Schema(title=TEXT(stored=True), content=TEXT) + +def create_index(articles): + """ + Creates the index. + """ + ix = create_in(indexdir, schema) + writer = ix.writer() + for article in articles: + writer.add_document(content=utils.clear_string(article.content)) + writer.commit() + +def search(term): + """ + Search for `term` in the index. + Returns a list of articles. + """ + try: + ix = open_dir(indexdir) + except (EmptyIndexError, OSError) as e: + raise EmptyIndexError + with ix.searcher() as searcher: + query = QueryParser("content", ix.schema).parse(term) + results = searcher.search(query, limit=None) + #return [(article["feed_id"], article["article_id"]) for article in results] + + +if __name__ == "__main__": + # Point of entry in execution mode. + #create_index() + print(nb_documents()) + results = search("Nothomb") + for article in results: + print(article) |