From fab5e5c782e906dfe27d716dd5684f6777799c36 Mon Sep 17 00:00:00 2001
From: Daoud Clarke <daoud.clarke@gmail.com>
Date: Sun, 8 Oct 2023 21:42:04 +0100
Subject: [PATCH] Use different dev and prod settings

---
 app/api.py                              |  2 +-
 app/apps.py                             | 19 ++++++++++++++++---
 app/{settings.py => settings_common.py} |  4 ----
 app/settings_dev.py                     |  5 +++++
 app/settings_prod.py                    |  5 +++++
 5 files changed, 27 insertions(+), 8 deletions(-)
 rename app/{settings.py => settings_common.py} (96%)
 create mode 100644 app/settings_dev.py
 create mode 100644 app/settings_prod.py

diff --git a/app/api.py b/app/api.py
index af959e9..713bfc3 100644
--- a/app/api.py
+++ b/app/api.py
@@ -1,9 +1,9 @@
 from multiprocessing import Queue
 from pathlib import Path
 
+from django.conf import settings
 from ninja import NinjaAPI
 
-from app import settings
 import mwmbl.crawler.app as crawler
 from mwmbl.indexer.batch_cache import BatchCache
 from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME
diff --git a/app/apps.py b/app/apps.py
index d4ad331..166aaff 100644
--- a/app/apps.py
+++ b/app/apps.py
@@ -1,12 +1,14 @@
-import os
 from multiprocessing import Process, Queue
+from pathlib import Path
 
 from django.apps import AppConfig
+from django.conf import settings
 
-from app import settings
 from app.api import queued_batches
 from mwmbl import background
+from mwmbl.indexer.paths import INDEX_NAME
 from mwmbl.indexer.update_urls import update_urls_continuously
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
 from mwmbl.url_queue import update_queue_continuously
 
 
@@ -15,7 +17,18 @@ class MwmblConfig(AppConfig):
     verbose_name = "Mwmbl Application"
 
     def ready(self):
-        if os.environ.get('RUN_MAIN') and settings.RUN_BACKGROUND_PROCESSES:
+        index_path = Path(settings.DATA_PATH) / INDEX_NAME
+        try:
+            existing_index = TinyIndex(item_factory=Document, index_path=index_path)
+            if existing_index.page_size != PAGE_SIZE or existing_index.num_pages != settings.NUM_PAGES:
+                raise ValueError(f"Existing index page sizes ({existing_index.page_size}) or number of pages "
+                                 f"({existing_index.num_pages}) do not match")
+        except FileNotFoundError:
+            print("Creating a new index")
+            TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=settings.NUM_PAGES,
+                             page_size=PAGE_SIZE)
+
+        if settings.RUN_BACKGROUND_PROCESSES:
             new_item_queue = Queue()
             Process(target=background.run, args=(settings.DATA_PATH,)).start()
             Process(target=update_queue_continuously, args=(new_item_queue, queued_batches,)).start()
diff --git a/app/settings.py b/app/settings_common.py
similarity index 96%
rename from app/settings.py
rename to app/settings_common.py
index 67f0d50..2753dc6 100644
--- a/app/settings.py
+++ b/app/settings_common.py
@@ -123,7 +123,3 @@ STATIC_URL = 'static/'
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
-# ===================== Custom Settings =========================
-
-DATA_PATH = "./devdata"
-RUN_BACKGROUND_PROCESSES = True
diff --git a/app/settings_dev.py b/app/settings_dev.py
new file mode 100644
index 0000000..bb8e33f
--- /dev/null
+++ b/app/settings_dev.py
@@ -0,0 +1,5 @@
+from app.settings_common import *
+
+DATA_PATH = "./devdata"
+RUN_BACKGROUND_PROCESSES = False
+NUM_PAGES = 2560
diff --git a/app/settings_prod.py b/app/settings_prod.py
new file mode 100644
index 0000000..37c9cf3
--- /dev/null
+++ b/app/settings_prod.py
@@ -0,0 +1,5 @@
+from app.settings_common import *
+
+DATA_PATH = "/app/storage"
+RUN_BACKGROUND_PROCESSES = True
+NUM_PAGES = 10240000