Use a separate page size for testing

performance.py

@@ -1,23 +1,23 @@
 """
 Test the performance of the search in terms of compression and speed.
 """
-import json
-import numpy as np
 import os
 from datetime import datetime
 from itertools import islice
 
+import numpy as np
 from spacy.lang.en import English
 from starlette.testclient import TestClient
 
-from app import app, complete
-from index import TinyIndexer, index_titles_and_urls, PAGE_SIZE, NUM_PAGES
+from app import app
+from index import TinyIndexer, index_titles_and_urls
 from paths import TEST_INDEX_PATH
 from wiki import get_wiki_titles_and_urls
 
-
 NUM_DOCUMENTS = 30000
 NUM_PAGES_FOR_STATS = 10
+TEST_PAGE_SIZE = 512
+TEST_NUM_PAGES = 32
 
 
 def query_test():
@@ -43,7 +43,7 @@ def query_test():
 
 def page_stats(indexer: TinyIndexer):
     page_sizes = []
-    for i in range(NUM_PAGES):
+    for i in range(TEST_NUM_PAGES):
         page = indexer.get_page(i)
         if page is not None:
             page_sizes.append(len(page))
@@ -57,7 +57,7 @@ def performance_test():
         os.remove(TEST_INDEX_PATH)
     except FileNotFoundError:
         print("No test index found, creating")
-    with TinyIndexer(TEST_INDEX_PATH, NUM_PAGES, PAGE_SIZE) as indexer:
+    with TinyIndexer(TEST_INDEX_PATH, TEST_NUM_PAGES, TEST_PAGE_SIZE) as indexer:
         titles_and_urls = get_wiki_titles_and_urls()
         titles_and_urls_slice = islice(titles_and_urls, NUM_DOCUMENTS)