Use different dev and prod settings

app/api.py

@@ -1,9 +1,9 @@
 from multiprocessing import Queue
 from pathlib import Path
 
+from django.conf import settings
 from ninja import NinjaAPI
 
-from app import settings
 import mwmbl.crawler.app as crawler
 from mwmbl.indexer.batch_cache import BatchCache
 from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME

app/apps.py

@@ -1,12 +1,14 @@
-import os
 from multiprocessing import Process, Queue
+from pathlib import Path
 
 from django.apps import AppConfig
+from django.conf import settings
 
-from app import settings
 from app.api import queued_batches
 from mwmbl import background
+from mwmbl.indexer.paths import INDEX_NAME
 from mwmbl.indexer.update_urls import update_urls_continuously
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
 from mwmbl.url_queue import update_queue_continuously
 
 
@@ -15,7 +17,18 @@ class MwmblConfig(AppConfig):
     verbose_name = "Mwmbl Application"
 
     def ready(self):
-        if os.environ.get('RUN_MAIN') and settings.RUN_BACKGROUND_PROCESSES:
+        index_path = Path(settings.DATA_PATH) / INDEX_NAME
+        try:
+            existing_index = TinyIndex(item_factory=Document, index_path=index_path)
+            if existing_index.page_size != PAGE_SIZE or existing_index.num_pages != settings.NUM_PAGES:
+                raise ValueError(f"Existing index page sizes ({existing_index.page_size}) or number of pages "
+                                 f"({existing_index.num_pages}) do not match")
+        except FileNotFoundError:
+            print("Creating a new index")
+            TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=settings.NUM_PAGES,
+                             page_size=PAGE_SIZE)
+
+        if settings.RUN_BACKGROUND_PROCESSES:
             new_item_queue = Queue()
             Process(target=background.run, args=(settings.DATA_PATH,)).start()
             Process(target=update_queue_continuously, args=(new_item_queue, queued_batches,)).start()

app/settings_common.py

@@ -123,7 +123,3 @@ STATIC_URL = 'static/'
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
-# ===================== Custom Settings =========================
-
-DATA_PATH = "./devdata"
-RUN_BACKGROUND_PROCESSES = True

app/settings_dev.py

@@ -0,0 +1,5 @@
+from app.settings_common import *
+
+DATA_PATH = "./devdata"
+RUN_BACKGROUND_PROCESSES = False
+NUM_PAGES = 2560

app/settings_prod.py

@@ -0,0 +1,5 @@
+from app.settings_common import *
+
+DATA_PATH = "/app/storage"
+RUN_BACKGROUND_PROCESSES = True
+NUM_PAGES = 10240000