diff --git a/mwmbl/background.py b/mwmbl/background.py index f1c1695..98e8eb5 100644 --- a/mwmbl/background.py +++ b/mwmbl/background.py @@ -24,6 +24,10 @@ def run(data_path: str, url_queue: Queue): url_db.create_tables() initialize_url_queue(url_queue) + try: + update_url_queue(url_queue) + except Exception: + logger.exception("Error updating URL queue") historical.run() index_path = Path(data_path) / INDEX_NAME batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME) diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.py index 3af9e44..9386680 100644 --- a/mwmbl/indexer/historical.py +++ b/mwmbl/indexer/historical.py @@ -4,7 +4,7 @@ from mwmbl.crawler.app import get_batches_for_date from mwmbl.database import Database from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase -DAYS = 66 +DAYS = 20 def run():