From 1b7c5723d26dbd2ac72361cf61d3ad66551458ab Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Fri, 17 May 2024 15:18:47 -0700 Subject: [PATCH 1/4] Don't deprovision within running Kolibri to create preseeded DBs --- Makefile | 7 ++++++- build_tools/preseed_home.py | 32 -------------------------------- 2 files changed, 6 insertions(+), 33 deletions(-) delete mode 100644 build_tools/preseed_home.py diff --git a/Makefile b/Makefile index df3aa492e5e..b078c93ae26 100644 --- a/Makefile +++ b/Makefile @@ -188,7 +188,12 @@ writeversion: @echo "Current version is now `cat kolibri/VERSION`" preseeddb: - PYTHONPATH=".:$PYTHONPATH" python build_tools/preseed_home.py + export KOLIBRI_HOME="$$(mktemp -d)"; \ + export PYTHONPATH=".:$$PYTHONPATH"; \ + cd kolibri; \ + python -m kolibri manage deprovision --destroy-all-user-data --permanent-irrevocable-data-loss; \ + mkdir -p dist/home; \ + cp $$KOLIBRI_HOME/*.sqlite3 dist/home/; setrequirements: rm -r requirements.txt || true # remove requirements.txt diff --git a/build_tools/preseed_home.py b/build_tools/preseed_home.py deleted file mode 100644 index d479ce4dfb0..00000000000 --- a/build_tools/preseed_home.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import shutil -import tempfile - -temphome = tempfile.mkdtemp() -os.environ["KOLIBRI_HOME"] = temphome - -from kolibri.main import initialize # noqa E402 -from kolibri.deployment.default.sqlite_db_names import ( # noqa E402 - ADDITIONAL_SQLITE_DATABASES, -) -from django.conf import settings # noqa E402 -from django.core.management import call_command # noqa E402 - -move_to = os.path.join(os.path.dirname(__file__), "..", "kolibri", "dist", "home") -shutil.rmtree(move_to, ignore_errors=True) -os.mkdir(move_to) - -print("Generating preseeded home data in {}".format(temphome)) - -initialize() -call_command( - "deprovision", "--destroy-all-user-data", "--permanent-irrevocable-data-loss" -) - -for db_config in settings.DATABASES.values(): - if db_config["ENGINE"] == "django.db.backends.sqlite3": - shutil.move(os.path.join(temphome, db_config["NAME"]), move_to) - -print("Moved all preseeded home data to {}".format(move_to)) - -shutil.rmtree(temphome) From ac8f733f78e6ca4afea142297d737cda1be7b3d3 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Mon, 20 May 2024 09:12:36 -0700 Subject: [PATCH 2/4] Try without python path; add rm of dist/home --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b078c93ae26..37e17b68ec7 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ writeversion: preseeddb: export KOLIBRI_HOME="$$(mktemp -d)"; \ - export PYTHONPATH=".:$$PYTHONPATH"; \ cd kolibri; \ + rm -rf dist/home; \ python -m kolibri manage deprovision --destroy-all-user-data --permanent-irrevocable-data-loss; \ mkdir -p dist/home; \ cp $$KOLIBRI_HOME/*.sqlite3 dist/home/; From 3fc8d81bdaf21844c1030ce6ca4cfb9623d04b99 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 21 May 2024 10:04:34 -0700 Subject: [PATCH 3/4] Convert to bash script with query verification --- Makefile | 7 +----- build_tools/preseed_home.sh | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) create mode 100755 build_tools/preseed_home.sh diff --git a/Makefile b/Makefile index 37e17b68ec7..c8479020109 100644 --- a/Makefile +++ b/Makefile @@ -188,12 +188,7 @@ writeversion: @echo "Current version is now `cat kolibri/VERSION`" preseeddb: - export KOLIBRI_HOME="$$(mktemp -d)"; \ - cd kolibri; \ - rm -rf dist/home; \ - python -m kolibri manage deprovision --destroy-all-user-data --permanent-irrevocable-data-loss; \ - mkdir -p dist/home; \ - cp $$KOLIBRI_HOME/*.sqlite3 dist/home/; + ./build_tools/preseed_home.sh setrequirements: rm -r requirements.txt || true # remove requirements.txt diff --git a/build_tools/preseed_home.sh b/build_tools/preseed_home.sh new file mode 100755 index 00000000000..c0b8ceae3dc --- /dev/null +++ b/build_tools/preseed_home.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -eux -o pipefail + +# Assumes this script is run from the project root +PROJECT_ROOT="$(pwd)" +KOLIBRI_HOME="$(mktemp -d)" +PYTHONPATH="${PYTHONPATH:-}" + +# Prepend project root to python path +if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH}" +else + PYTHONPATH="${PROJECT_ROOT}" +fi + +export KOLIBRI_HOME +export PYTHONPATH + +# Clean existing DBs and generate fresh DBs +rm -rf kolibri/dist/home +(cd kolibri && python -m kolibri manage deprovision --destroy-all-user-data --permanent-irrevocable-data-loss) +mkdir -p kolibri/dist/home +cp $KOLIBRI_HOME/*.sqlite3 kolibri/dist/home/ +rm -rf $KOLIBRI_HOME + +# disable command echoing +set +x + +# Verify DBs +declare -A QUERY_MATRIX +QUERY_MATRIX["select count(*) from morango_databaseidmodel;"]="db.sqlite3" +QUERY_MATRIX["select count(*) from morango_instanceidmodel;"]="db.sqlite3" +QUERY_MATRIX["select count(*) from discovery_networklocation;"]="networklocation.sqlite3" +QUERY_MATRIX["select count(*) from jobs;"]="job_storage.sqlite3" + +echo "Verifying databases..." + +for QUERY in "${!QUERY_MATRIX[@]}"; do + DB=${QUERY_MATRIX[$QUERY]} + ROW_COUNT=$(sqlite3 -cmd ".headers off" kolibri/dist/home/$DB "$QUERY") + if [ "$ROW_COUNT" -ne 0 ]; then + echo "Preseeded DBs have existing data | $ROW_COUNT = $QUERY" + exit 1 + fi +done + +echo "Done! Preseeded databases generated" From 641e80474a556f1e7f27d844dab1341af68c195c Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 21 May 2024 10:28:01 -0700 Subject: [PATCH 4/4] Install sqlite3 dep --- .github/workflows/build_whl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_whl.yml b/.github/workflows/build_whl.yml index 958161f2f13..48aa3dee240 100644 --- a/.github/workflows/build_whl.yml +++ b/.github/workflows/build_whl.yml @@ -31,7 +31,7 @@ jobs: - name: Install Ubuntu dependencies run: | apt-get -y -qq update - apt-get install -y gettext sudo + apt-get install -y gettext sudo sqlite3 - name: Use Node.js uses: actions/setup-node@v4 with: