Make data in ./run_local.sh persistent across reloads and support par…

…ameter DBs

diracx-testing/src/diracx/testing/__init__.py

@@ -142,18 +142,6 @@ def __call__(self):
         )
 
 
-def fake_available_implementations(name, *, real_available_implementations):
-    from diracx.testing.mock_osdb import MockOSDBMixin
-
-    implementations = real_available_implementations(name)
-
-    # Dynamically generate a class that inherits from the first implementation
-    # but that also has the MockOSDBMixin
-    MockParameterDB = type(name, (MockOSDBMixin, implementations[0]), {})
-
-    return [MockParameterDB] + implementations
-
-
 class ClientFactory:
 
     def __init__(
@@ -200,7 +188,7 @@ def enrich_tokens(access_payload: dict, refresh_payload: dict):
             for e in select_from_extension(group="diracx.db.os")
         }
         BaseOSDB.available_implementations = partial(
-            fake_available_implementations,
+            fake_available_osdb_implementations,
             real_available_implementations=BaseOSDB.available_implementations,
         )
 

diracx-testing/src/diracx/testing/mock_osdb.py

@@ -1,5 +1,10 @@
 from __future__ import annotations
 
+__all__ = (
+    "MockOSDBMixin",
+    "fake_available_osdb_implementations",
+)
+
 import contextlib
 from datetime import datetime, timezone
 from functools import partial
@@ -146,3 +151,13 @@ async def search(
 
     async def ping(self):
         return await self._sql_db.ping()
+
+
+def fake_available_osdb_implementations(name, *, real_available_implementations):
+    implementations = real_available_implementations(name)
+
+    # Dynamically generate a class that inherits from the first implementation
+    # but that also has the MockOSDBMixin
+    MockParameterDB = type(name, (MockOSDBMixin, implementations[0]), {})
+
+    return [MockParameterDB] + implementations

diracx-testing/src/diracx/testing/routers.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from functools import partial
+
+from diracx.testing.mock_osdb import fake_available_osdb_implementations
+
+
+@asynccontextmanager
+async def ensure_dbs_exist():
+    from diracx.db.__main__ import init_os, init_sql
+
+    await init_sql()
+    await init_os()
+    yield
+
+
+def create_app():
+    """Create a FastAPI application for testing purposes.
+
+    This is a wrapper around diracx.routers.create_app that:
+     * adds a lifetime function to ensure the DB schemas are initialized
+     * replaces the parameter DBs with sqlite-backed versions
+    """
+    from diracx.db.os.utils import BaseOSDB
+    from diracx.routers import create_app
+
+    BaseOSDB.available_implementations = partial(
+        fake_available_osdb_implementations,
+        real_available_implementations=BaseOSDB.available_implementations,
+    )
+
+    app = create_app()
+    app.lifetime_functions.append(ensure_dbs_exist)
+
+    return app

run_local.sh

@@ -27,11 +27,14 @@ dirac internal add-user "${tmp_dir}/cs_store/initialRepo" \
   --preferred-username=localuser
 
 export DIRACX_CONFIG_BACKEND_URL="git+file://${tmp_dir}/cs_store/initialRepo"
-export DIRACX_DB_URL_AUTHDB="sqlite+aiosqlite:///:memory:"
-export DIRACX_DB_URL_JOBDB="sqlite+aiosqlite:///:memory:"
-export DIRACX_DB_URL_JOBLOGGINGDB="sqlite+aiosqlite:///:memory:"
-export DIRACX_DB_URL_SANDBOXMETADATADB="sqlite+aiosqlite:///:memory:"
-export DIRACX_DB_URL_TASKQUEUEDB="sqlite+aiosqlite:///:memory:"
+export DIRACX_DB_URL_AUTHDB="sqlite+aiosqlite:///${tmp_dir}/authdb.db"
+export DIRACX_DB_URL_JOBDB="sqlite+aiosqlite:///${tmp_dir}/jobdb.db"
+export DIRACX_DB_URL_JOBLOGGINGDB="sqlite+aiosqlite:///${tmp_dir}/jobloggingdb.db"
+export DIRACX_DB_URL_SANDBOXMETADATADB="sqlite+aiosqlite:///${tmp_dir}/sandboxmetadatadb.db"
+export DIRACX_DB_URL_TASKQUEUEDB="sqlite+aiosqlite:///${tmp_dir}/taskqueuedb.db"
+# This script monkey patches the parameter db to use a sqlite database rather
+# than requiring a full opensearch instance so we use a sqlalchmey dsn here
+export DIRACX_OS_DB_JOBPARAMETERSDB='{"sqlalchemy_dsn": "sqlite+aiosqlite:///'${tmp_dir}'/jobparametersdb.db"}'
 export DIRACX_SERVICE_AUTH_TOKEN_KEY="file://${signing_key}"
 export DIRACX_SERVICE_AUTH_STATE_KEY="${state_key}"
 export DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS='["http://'$(hostname| tr -s '[:upper:]' '[:lower:]')':8000/docs/oauth2-redirect"]'
@@ -41,7 +44,7 @@ export DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS='{"endpoint_url": "http://localhost
 
 moto_server -p3000 &
 moto_pid=$!
-uvicorn --factory diracx.routers:create_app --reload &
+uvicorn --factory diracx.testing.routers:create_app --reload &
 diracx_pid=$!
 
 success=0