From c84eeba92e535f81230f10643d238d6aaf29be37 Mon Sep 17 00:00:00 2001 From: Daoud Clarke Date: Fri, 16 Apr 2021 22:01:01 +0100 Subject: [PATCH] Use a separate page size for testing --- performance.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.py b/performance.py index 2d4280d..3376198 100644 --- a/performance.py +++ b/performance.py @@ -1,23 +1,23 @@ """ Test the performance of the search in terms of compression and speed. """ -import json -import numpy as np import os from datetime import datetime from itertools import islice +import numpy as np from spacy.lang.en import English from starlette.testclient import TestClient -from app import app, complete -from index import TinyIndexer, index_titles_and_urls, PAGE_SIZE, NUM_PAGES +from app import app +from index import TinyIndexer, index_titles_and_urls from paths import TEST_INDEX_PATH from wiki import get_wiki_titles_and_urls - NUM_DOCUMENTS = 30000 NUM_PAGES_FOR_STATS = 10 +TEST_PAGE_SIZE = 512 +TEST_NUM_PAGES = 32 def query_test(): @@ -43,7 +43,7 @@ def query_test(): def page_stats(indexer: TinyIndexer): page_sizes = [] - for i in range(NUM_PAGES): + for i in range(TEST_NUM_PAGES): page = indexer.get_page(i) if page is not None: page_sizes.append(len(page)) @@ -57,7 +57,7 @@ def performance_test(): os.remove(TEST_INDEX_PATH) except FileNotFoundError: print("No test index found, creating") - with TinyIndexer(TEST_INDEX_PATH, NUM_PAGES, PAGE_SIZE) as indexer: + with TinyIndexer(TEST_INDEX_PATH, TEST_NUM_PAGES, TEST_PAGE_SIZE) as indexer: titles_and_urls = get_wiki_titles_and_urls() titles_and_urls_slice = islice(titles_and_urls, NUM_DOCUMENTS)