beenull/mwmbl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Update the URL queue earlier

Browse source
This commit is contained in:
Daoud Clarke 2022-12-31 23:37:59 +00:00
parent 0d33b4f68f
commit 7c14cd99f8
2 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
mwmbl/background.py
View file

@ -24,6 +24,10 @@ def run(data_path: str, url_queue: Queue):
url_db.create_tables() url_db.create_tables()
initialize_url_queue(url_queue) initialize_url_queue(url_queue)
try:
update_url_queue(url_queue)
except Exception:
logger.exception("Error updating URL queue")
historical.run() historical.run()
index_path = Path(data_path) / INDEX_NAME index_path = Path(data_path) / INDEX_NAME
batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME) batch_cache = BatchCache(Path(data_path) / BATCH_DIR_NAME)

2
mwmbl/indexer/historical.py
View file

@ -4,7 +4,7 @@ from mwmbl.crawler.app import get_batches_for_date
from mwmbl.database import Database from mwmbl.database import Database
from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase from mwmbl.indexer.indexdb import BatchInfo, BatchStatus, IndexDatabase
DAYS = 66 DAYS = 20
def run(): def run():