From a097ec9fbe889285eda5ca47db43ef00e13bcc39 Mon Sep 17 00:00:00 2001
From: Daoud Clarke <daoud.clarke@gmail.com>
Date: Mon, 18 Jul 2022 23:42:09 +0100
Subject: [PATCH] Allow more tries so that popular terms can be indexed
---
mwmbl/background.py | 18 +++++++++---------
mwmbl/indexer/update_pages.py | 5 ++---
2 files changed, 11 insertions(+), 12 deletions(-)
diff --git a/mwmbl/background.py b/mwmbl/background.py
index 5feb064..89f4829 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -13,16 +13,16 @@ logger = getLogger(__name__)
def run(index_path: str):
- # historical.run()
+ historical.run()
while True:
- # try:
- # retrieve_batches()
- # except Exception:
- # logger.exception("Error retrieving batches")
- # try:
- # run_preprocessing(index_path)
- # except Exception:
- # logger.exception("Error preprocessing")
+ try:
+ retrieve_batches()
+ except Exception:
+ logger.exception("Error retrieving batches")
+ try:
+ run_preprocessing(index_path)
+ except Exception:
+ logger.exception("Error preprocessing")
try:
run_update(index_path)
except Exception:
diff --git a/mwmbl/indexer/update_pages.py b/mwmbl/indexer/update_pages.py
index 7b73541..b04c421 100644
--- a/mwmbl/indexer/update_pages.py
+++ b/mwmbl/indexer/update_pages.py
@@ -17,14 +17,13 @@ def run_update(index_path):
with TinyIndex(Document, index_path, 'w') as indexer:
with Database() as db:
index_db = IndexDatabase(db.connection)
- # pages_to_process = index_db.get_queued_pages()
- pages_to_process = [278107]
+ pages_to_process = index_db.get_queued_pages()
print(f"Got {len(pages_to_process)} pages to process")
for i in pages_to_process:
documents = index_db.get_queued_documents_for_page(i)
print(f"Documents queued for page {i}: {len(documents)}")
if len(documents) > 0:
- for j in range(3):
+ for j in range(20):
try:
indexer.add_to_page(i, documents)
break