diff --git a/mwmbl/background.py b/mwmbl/background.py index f1c16958f32cc7d4f0360d6f2d36cc90d98acd55..98e8eb50d845a5bb1b427eaf4cd58769188062cc 100644 --- a/mwmbl/background.py +++ b/mwmbl/background.py @@ -24,6 +24,10 @@ def run(data_path: str, url_queue: Queue): url_db.create_tables() initialize_url_queue(url_queue) + try: + update_url_queue(url_queue) + except Exception: + logger.exception("Error updating URL queue") historical.run() index_path = Path(data_path) / INDEX_NAME batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME) diff --git a/mwmbl/indexer/historical.py b/mwmbl/indexer/historical.py index 3af9e4447608194cedd17aed9e5bcec7135982d0..938668041a1cb46be3d123ec6142c281cca3ebf2 100644 --- a/mwmbl/indexer/historical.py +++ b/mwmbl/indexer/historical.py @@ -4,7 +4,7 @@ from mwmbl.crawler.app import get_batches_for_date from mwmbl.database import Database from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase -DAYS = 66 +DAYS = 20 def run():