Update the URL queue earlier

mwmbl/background.py

@@ -24,6 +24,10 @@ def run(data_path: str, url_queue: Queue):
         url_db.create_tables()
 
     initialize_url_queue(url_queue)
+    try:
+        update_url_queue(url_queue)
+    except Exception:
+        logger.exception("Error updating URL queue")
     historical.run()
     index_path = Path(data_path) / INDEX_NAME
     batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)

mwmbl/indexer/historical.py

@@ -4,7 +4,7 @@ from mwmbl.crawler.app import get_batches_for_date
 from mwmbl.database import Database
 from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase
 
-DAYS = 66
+DAYS = 20
 
 
 def run():