Populate development dbs with test fixture data

Closes #2313.

It was suggested in #2313 that using existing test fixture data as dummy data for local development purposes may provide a good alternative to using a copy of prod databases.

The approach taken here shows one way to populate local dbs with test fixture data, based on a comment on the original issue, and a spike (#2407).

Co-authored-by: Jon Massey <jon.massey@thedatalab.org>

Author: KatieB5

coding_systems/versioning/models.py

@@ -76,13 +76,19 @@ def update_coding_system_database_connections():
         for coding_system_release in CodingSystemRelease.objects.all():
             if not CODING_SYSTEMS[coding_system_release.coding_system].has_database:
                 continue
-            db_path = (
-                settings.CODING_SYSTEMS_DATABASE_DIR
-                / coding_system_release.coding_system
-                / f"{coding_system_release.database_alias}.sqlite3"
-            )
+            db_path = build_db_path(coding_system_release)
             database_dict = {
                 **connections.databases[DEFAULT_DB_ALIAS],
                 **dj_database_url.parse(f"sqlite:///{db_path}"),
             }
             connections.databases[coding_system_release.database_alias] = database_dict
+
+
+def build_db_path(coding_system_release):
+    db_path = (
+        settings.CODING_SYSTEMS_DATABASE_DIR
+        / coding_system_release.coding_system
+        / f"{coding_system_release.database_alias}.sqlite3"
+    )
+
+    return db_path

justfile

@@ -247,6 +247,41 @@ assets-test: assets-install
     npm run test:coverage
 
 
+# Build a lightweight local development setup using test fixture data.
+build-dbs-for-local-development nuclear="":
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    # WARNING: Passing 'nuclear' to this just recipe will create a
+    # backup copy of your current local db, if it exists, then create a
+    # new empty core `db.sqlite3` in the root directory of your
+    # opencodelists project folder.
+    if [ -n "{{ nuclear }}" ]; then
+        echo "Nuclear option enabled: deleting db.sqlite3"
+
+        if [[ -z "${DATABASE_URL:-}" ]]; then
+            CORE_DB_PATH="db.sqlite3"
+            else
+            CORE_DB_PATH="${DATABASE_URL/sqlite:\/\/\//}"
+            fi
+        mv "$CORE_DB_PATH" "$CORE_DB_PATH.backup"
+
+        # Set up or update the local dev environment:
+        # - Recreates core DB and applies migrations
+        just dev-setup
+
+        # Run custom command to:
+        # - Load CodingSystemReleases needed for the test data fixtures
+        # - Remove old coding system release dbs (with confirmation)
+        # - Create and migrate new coding system release dbs
+        # - Load test data into coding system release dbs
+        $BIN/python manage.py setup_local_dev_databases
+    else
+        echo "Skipping db.sqlite3 deletion. Run with 'nuclear' parameter to enable."
+    fi
+
+
+
 # build docker image env=dev|prod
 docker-build env="dev": _env
     {{ just_executable() }} docker/build {{ env }}

opencodelists/management/commands/setup_local_dev_databases.py

@@ -0,0 +1,103 @@
+from django.core.management import BaseCommand, call_command
+
+from coding_systems.versioning.models import (
+    CodingSystemRelease,
+    build_db_path,
+    update_coding_system_database_connections,
+)
+from opencodelists.tests.fixtures import build_fixtures
+
+
+class Command(BaseCommand):
+    help = """Populates local opencodelists dbs with test fixture data: Loads the CodingSystemReleases needed for the snomed and dmd fixtures into the Core db (you will be prompted for consent to do this), deletes existing test coding system release dbs, creates new coding system release dbs, populates these with test fixture data, instantiates a universe of fixture objects, and creates a superuser. Running this command directly is not recommended, instead use `just build-dbs-for-local-development`"""
+
+    def handle(self, *args, **options):
+        load_versioning_fixture_answer = input(
+            "Running this command will load the CodingSystemReleases needed for the snomed and dmd test fixtures, into the Core db. Any local test coding system release dbs will then be deleted and recreated. Do you want to proceed? (Y/n)"
+        )
+
+        if load_versioning_fixture_answer == "n":
+            return
+
+        call_command(
+            "loaddata",
+            "coding_systems/versioning/fixtures/coding_system_releases.json",
+        )
+
+        self.stdout.write("Deleting local coding system release dbs...")
+        for coding_system_release in CodingSystemRelease.objects.all():
+            db_path = build_db_path(coding_system_release)
+            if db_path.exists():
+                db_path.unlink()
+                self.stdout.write(f"{str(db_path)} has been deleted")
+
+        self.stdout.write(
+            "Migrating coding system release dbs and populating them with test fixture data"
+        )
+
+        update_coding_system_database_connections()
+
+        # migrate snomedct test db and load test fixtures
+        call_command("migrate", "snomedct", database="snomedct_test_20200101")
+
+        call_command(
+            "loaddata",
+            "coding_systems/snomedct/fixtures/core-model-components.snomedct_test_20200101.json",
+            database="snomedct_test_20200101",
+        )
+
+        call_command(
+            "loaddata",
+            "coding_systems/snomedct/fixtures/tennis-elbow.snomedct_test_20200101.json",
+            database="snomedct_test_20200101",
+        )
+
+        call_command(
+            "loaddata",
+            "coding_systems/snomedct/fixtures/tennis-toe.snomedct_test_20200101.json",
+            database="snomedct_test_20200101",
+        )
+
+        # migrate dmd test db and load test fixture
+        call_command("migrate", "dmd", database="dmd_test_20200101")
+
+        call_command(
+            "loaddata",
+            "coding_systems/dmd/fixtures/asthma-medication.dmd_test_20200101.json",
+            database="dmd_test_20200101",
+        )
+
+        # migrate bnf test db and load test fixture
+        call_command("migrate", "bnf", database="bnf_test_20200101")
+
+        call_command(
+            "loaddata",
+            "coding_systems/bnf/fixtures/asthma.bnf_test_20200101.json",
+            database="bnf_test_20200101",
+        )
+
+        # migrate icd10 test db and load test fixture
+        call_command("migrate", "icd10", database="icd10_test_20200101")
+        call_command(
+            "loaddata",
+            "coding_systems/icd10/fixtures/icd10.icd10_test_20200101.json",
+            database="icd10_test_20200101",
+        )
+
+        # instantiate the test data universe
+        build_fixtures()
+
+        call_command(
+            "createsuperuser",
+            no_input=True,
+            username="localdev",
+            email="localdev@example.com",
+        )
+
+        self.stdout.write(
+            "Local setup complete! You can now:\n"
+            " - Log in as `localdev`\n"
+            " - Search for 'arthritis', 'tennis', and 'elbow'\n"
+            " - Build codelists with the concepts returned from these searches (see /opencodelists/opencodelists/tests/fixtures.py for more info)\n"
+            " - View a BNF codelist, a minimal codelist, and a new-style SNOMED CT codelist"
+        )