0ct0pu5/mwmbl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use a separate page size for testing

Browse source
This commit is contained in:
Daoud Clarke 2021-04-16 22:01:01 +01:00
parent ced0fceae8
commit c84eeba92e
1 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
performance.py
View file

@ -1,23 +1,23 @@
"""
Test the performance of the search in terms of compression and speed.
"""
import json
import numpy as np
import os
from datetime import datetime
from itertools import islice
import numpy as np
from spacy.lang.en import English
from starlette.testclient import TestClient
from app import app, complete
from index import TinyIndexer, index_titles_and_urls, PAGE_SIZE, NUM_PAGES
from app import app
from index import TinyIndexer, index_titles_and_urls
from paths import TEST_INDEX_PATH
from wiki import get_wiki_titles_and_urls
NUM_DOCUMENTS = 30000
NUM_PAGES_FOR_STATS = 10
TEST_PAGE_SIZE = 512
TEST_NUM_PAGES = 32
def query_test():
@ -43,7 +43,7 @@ def query_test():
def page_stats(indexer: TinyIndexer):
page_sizes = []
for i in range(NUM_PAGES):
for i in range(TEST_NUM_PAGES):
page = indexer.get_page(i)
if page is not None:
page_sizes.append(len(page))
@ -57,7 +57,7 @@ def performance_test():
os.remove(TEST_INDEX_PATH)
except FileNotFoundError:
print("No test index found, creating")
with TinyIndexer(TEST_INDEX_PATH, NUM_PAGES, PAGE_SIZE) as indexer:
with TinyIndexer(TEST_INDEX_PATH, TEST_NUM_PAGES, TEST_PAGE_SIZE) as indexer:
titles_and_urls = get_wiki_titles_and_urls()
titles_and_urls_slice = islice(titles_and_urls, NUM_DOCUMENTS)