Add CI workflow for testing db migrations (#2326)

* Add workflows/db.yml and wip script for testing db migrations Signed-off-by: wslulciuc <willy@datakin.com> * Add --args to ./docker/up.sh Signed-off-by: wslulciuc <willy@datakin.com> * Add db-data volume Signed-off-by: wslulciuc <willy@datakin.com> * Add .github/workflows/db.sh Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add .github/workflows/db.sh Signed-off-by: wslulciuc <willy@datakin.com> * db.sh -> db-migration.sh Signed-off-by: wslulciuc <willy@datakin.com> * Use git sha1 for tag on `--build` Signed-off-by: wslulciuc <willy@datakin.com> * Use `seed_api` for docker service name and build on `--build` Signed-off-by: wslulciuc <willy@datakin.com> * Remove cd into root dir in docker/prune.sh Signed-off-by: wslulciuc <willy@datakin.com> * Cleanup docker/volumes.sh Signed-off-by: wslulciuc <willy@datakin.com> * Fix log error formatting Signed-off-by: wslulciuc <willy@datakin.com> * Add db-backup volume Signed-off-by: wslulciuc <willy@datakin.com> * Add `--no-web` and `--no-volumes` flags Signed-off-by: wslulciuc <willy@datakin.com> * Add `docker-compose.web-dev.yml` to support `--no-web` flag Signed-off-by: wslulciuc <willy@datakin.com> * Add `log_db_migration()` to `db-migration.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Add comments and cleanup logic in `docker/up.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Use `sudo` to run `db-migration.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Fix logic for `--no-web` and `--no-volumes` Signed-off-by: wslulciuc <willy@datakin.com> * continued: Fix logic for `--no-web` and `--no-volumes` Signed-off-by: wslulciuc <willy@datakin.com> * Remove `-it` flags from log_db_migration() Signed-off-by: wslulciuc <willy@datakin.com> * Set image in docker-compose.web-dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * Use dev compose file in build mode when seeding http api server Signed-off-by: wslulciuc <willy@datakin.com> * Set default value for `--build` Signed-off-by: wslulciuc <willy@datakin.com> * Append args Signed-off-by: wslulciuc <willy@datakin.com> * Cleanup image tags in dev mode Signed-off-by: wslulciuc <willy@datakin.com> * More debugging... Signed-off-by: wslulciuc <willy@datakin.com> * Set `DOCKER_BUILDKIT` and reorder appending compose files Signed-off-by: wslulciuc <willy@datakin.com> * Add migrate-db CI job Signed-off-by: wslulciuc <willy@datakin.com> * Cleanup debugging Signed-off-by: wslulciuc <willy@datakin.com> * Fix path to db-migration.sh Signed-off-by: wslulciuc <willy@datakin.com> * Remove deps for build-api CI job Signed-off-by: wslulciuc <willy@datakin.com> * continued: Remove deps for build-api CI job Signed-off-by: wslulciuc <willy@datakin.com> * Limit output from db-migration.sh Signed-off-by: wslulciuc <willy@datakin.com> * Print entire flyway_schema_history row Signed-off-by: wslulciuc <willy@datakin.com> * Install docker compose in migrate-db CI job Signed-off-by: wslulciuc <willy@datakin.com> * Ensure migrate-db CI jobs fails on migration issues Signed-off-by: wslulciuc <willy@datakin.com> * Order db migrations by installed_on Signed-off-by: wslulciuc <willy@datakin.com> * Update comments Signed-off-by: wslulciuc <willy@datakin.com> * Update chart docs to use docker-compose.db.yml Signed-off-by: wslulciuc <willy@datakin.com> * Rename db migration backup container Signed-off-by: wslulciuc <willy@datakin.com> * Add additional constants and comments to `db-migration.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Apply minor formatting Signed-off-by: wslulciuc <willy@datakin.com> * continued: Apply minor formatting Signed-off-by: wslulciuc <willy@datakin.com> * continued: Apply minor formatting Signed-off-by: wslulciuc <willy@datakin.com> * continued: Apply minor formatting Signed-off-by: wslulciuc <willy@datakin.com> * continued: Apply minor formatting Signed-off-by: wslulciuc <willy@datakin.com> * Always printout docker container logs for debugging Signed-off-by: wslulciuc <willy@datakin.com> * Replace `latest` with `0.29.0` in `db-migration.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Log simple log on migration error Signed-off-by: wslulciuc <willy@datakin.com> * Simplify script output for readability Signed-off-by: wslulciuc <willy@datakin.com> * continued: Simplify script output for readability Signed-off-by: wslulciuc <willy@datakin.com> * Add docs to `db-migration.sh` Signed-off-by: wslulciuc <willy@datakin.com> Signed-off-by: wslulciuc <willy@datakin.com>
MarquezProject · Jan 9, 2023 · 9a481bd · 9a481bd
1 parent 71ceaab
commit 9a481bd
Show file tree

Hide file tree

Showing 13 changed files with 256 additions and 67 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -148,6 +148,15 @@ jobs:
       - run: npm install --prefix=${HOME}/.local --global redoc-cli
       - run: redoc-cli bundle spec/openapi.yml
 
+  migrate-db:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:current
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/db-migration.sh
+
   release-java:
     working_directory: ~/marquez
     machine:
@@ -195,6 +204,9 @@ workflows:
       - unit-test-web
       - unit-test-client-python
       - lint-spec-api
+      - migrate-db:
+          requires:
+            - build-api
   release:
     jobs:
       - build-client-python:

diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Copyright 2018-2022 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# A script used in CI to test database migrations by:
+#   (1) Applying db migrations on latest Marquez release
+#   (2) Take a backup of db from Step 1
+#   (3) Applying db migrations on latest Marquez build using backup
+#
+# Usage: $ ./db-migration.sh
+
+# Version of PostgreSQL
+readonly POSTGRES_VERSION="12.1"
+# Version of Marquez
+readonly MARQUEZ_VERSION="0.29.0"
+# Build version of Marquez
+readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
+
+readonly DB_MIGRATION_VOLUME="marquez_db-backup"
+readonly DB_MIGRATION_BACKUP="db-migration-backup"
+readonly DB_MIGRATION_QUERY=$(cat <<-END
+  SELECT version,installed_on,checksum
+    FROM flyway_schema_history
+   WHERE version IS NOT NULL
+   ORDER BY installed_on DESC LIMIT 1;
+END
+)
+
+log() {
+  echo -e "\033[1m>>\033[0m ${1}"
+}
+
+error() {
+  echo -e "\033[0;31merror: ${1}\033[0m"
+}
+
+exit_with_cause() {
+  log "please view container logs for more details on cause:"
+  docker-compose logs
+  exit 1
+}
+
+query_db_migration() {
+  # Start db using backup
+  [[ $(docker ps -f "name=${DB_MIGRATION_BACKUP}" --format '{{.Names}}') == "${DB_MIGRATION_BACKUP}" ]] || \
+    docker run -d --name "${DB_MIGRATION_BACKUP}" \
+      -v "${DB_MIGRATION_VOLUME}:/var/lib/postgresql/data" \
+      "postgres:${POSTGRES_VERSION}"
+  # Query applied db migrations
+  log "latest migration applied to db:"
+  docker exec "${DB_MIGRATION_BACKUP}" \
+    psql -U marquez -c "${DB_MIGRATION_QUERY}"
+}
+
+# Change working directory to project root
+project_root=$(git rev-parse --show-toplevel)
+cd "${project_root}/"
+
+# (1) Apply db migrations on latest Marquez release
+log "start db with latest migrations (marquez=${MARQUEZ_VERSION}):"
+if ! ./docker/up.sh \
+  --args "--exit-code-from seed_marquez" \
+  --tag "${MARQUEZ_VERSION}" \
+  --no-web \
+  --seed > /dev/null; then
+  error "failed to start db using backup!"
+  exit_with_cause
+fi
+
+# Query, then display schema migration applied
+query_db_migration
+
+# (2) Apply db migrations on latest Marquez build using backup
+log "start db using backup (marquez=${MARQUEZ_BUILD_VERSION}):"
+if ! ./docker/up.sh \
+  --args "--exit-code-from seed_marquez" \
+  --no-web \
+  --no-volumes \
+  --build \
+  --seed > /dev/null; then
+  error "failed to start db using backup!"
+  exit_with_cause
+fi
+
+# Query, then display additional schema migration applied on backup (if any)
+query_db_migration
+
+log "DONE!"
diff --git a/.circleci/get-docker-compose.sh b/.circleci/get-docker-compose.sh
@@ -1,24 +1,18 @@
 #!/bin/bash
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright 2018-2022 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
 #
 # Usage: $ ./get-docker-compose.sh
 
 set -e
 
-curl -L https://github.com/docker/compose/releases/download/1.25.3/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
-chmod +x ~/docker-compose
-sudo mv ~/docker-compose /usr/local/bin/docker-compose
-docker-compose --version
+# Download docker compose
+curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > ~/docker-compose
+
+# Change permissions, relocate docker compose, then verify
+chmod +x ~/docker-compose && \
+  sudo mv ~/docker-compose /usr/local/bin/docker-compose && \
+  docker-compose --version
 
-echo "DONE!"
+echo "DONE!"
diff --git a/chart/README.md b/chart/README.md
@@ -137,7 +137,7 @@ Contents of the ```./../docker-compose-postgres..yml``` file can be customized
 to better represent your desired setup.
 
 ```bash
-docker-compose -f ./../docker-compose.postgres.yml -p marquez-postgres up
+docker-compose -f ./../docker-compose.db.yml -p marquez-postgres up
 ```
 
 Once the Postgres instance has been created, run the following command to locate the IP

diff --git a/docker-compose.postgres.yml → docker-compose.db.yml b/docker-compose.postgres.yml → docker-compose.db.yml
@@ -2,6 +2,7 @@ version: "3.7"
 services:
   db:
     image: postgres:12.1
+    container_name: marquez-db
     ports:
       - "5432:5432"
     environment:

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -1,13 +1,10 @@
 version: "3.7"
 services:
   api:
-    image: "marquezproject/marquez:${TAG}"
     build: .
 
-  web:
-    build:
-      context: ./web
-      dockerfile: Dockerfile
+  seed_marquez:
+    build: .
 
   pghero:
     image: ankane/pghero

diff --git a/docker-compose.web-dev.yml b/docker-compose.web-dev.yml
@@ -0,0 +1,6 @@
+version: "3.7"
+services:
+  web:
+    build:
+      context: ./web
+      dockerfile: Dockerfile
diff --git a/docker-compose.web.yml b/docker-compose.web.yml
@@ -0,0 +1,12 @@
+version: "3.7"
+services:
+  web:
+    image: "marquezproject/marquez-web:${TAG}"
+    container_name: marquez-web
+    environment:
+      - MARQUEZ_HOST=api
+      - MARQUEZ_PORT=${API_PORT}
+    ports:
+      - "${WEB_PORT}:${WEB_PORT}"
+    depends_on:
+      - api
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,26 +10,13 @@ services:
       - "${API_PORT}:${API_PORT}"
       - "${API_ADMIN_PORT}:${API_ADMIN_PORT}"
     volumes:
-      - utils:/opt/marquez
+      - data:/opt/marquez
     links:
       - "db:postgres"
     depends_on:
       - db
     entrypoint: ["/opt/marquez/wait-for-it.sh", "db:5432", "--", "./entrypoint.sh"]
 
-  web:
-    image: "marquezproject/marquez-web:${TAG}"
-    container_name: marquez-web
-    environment:
-      - MARQUEZ_HOST=api
-      - MARQUEZ_PORT=${API_PORT}
-    ports:
-      - "${WEB_PORT}:${WEB_PORT}"
-    stdin_open: true
-    tty: true
-    depends_on:
-      - api
-
   db:
     image: postgres:12.1
     container_name: marquez-db
@@ -42,12 +29,15 @@ services:
       - MARQUEZ_USER=marquez
       - MARQUEZ_PASSWORD=marquez
     volumes:
-      - ./docker/postgresql.conf:/etc/postgresql/postgresql.conf
+      - db-conf:/etc/postgresql
       - db-init:/docker-entrypoint-initdb.d
+      - db-backup:/var/lib/postgresql/data
     command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
     # Enables SQL statement logging (see: https://www.postgresql.org/docs/12/runtime-config-logging.html#GUC-LOG-STATEMENT)
     # command: ["postgres", "-c", "log_statement=all"]
 
 volumes:
-  utils:
+  data:
+  db-conf:
   db-init:
+  db-backup:
diff --git a/docker/prune.sh b/docker/prune.sh
@@ -7,8 +7,4 @@
 
 set -e
 
-# Change working directory to project root
-project_root=$(git rev-parse --show-toplevel)
-cd "${project_root}"
-
 docker image prune -a --filter "until=24h" --force
diff --git a/docker/up.sh b/docker/up.sh
@@ -2,18 +2,22 @@
 #
 # Copyright 2018-2022 contributors to the Marquez project
 # SPDX-License-Identifier: Apache-2.0
+#
+# Usage: $ ./build-and-push.sh [FLAGS] [ARG...]
 
 set -e
 
-VERSION=0.29.0
-DOCKER_DIR=$(dirname $0)
+# Version of Marquez
+readonly VERSION=0.29.0
+# Build version of Marquez
+readonly BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
 
 title() {
   echo -e "\033[1m${1}\033[0m"
 }
 
 usage() {
-  echo "usage: ./$(basename -- ${0}) [--api-port PORT] [--web-port PORT] [--tag TAG] [--build] [--seed] [--detach]"
+  echo "usage: ./$(basename -- ${0}) [FLAGS] [ARG...]"
   echo "A script used to run Marquez via Docker"
   echo
   title "EXAMPLES:"
@@ -36,12 +40,15 @@ usage() {
   echo "  -a, --api-port int          api port (default: 5000)"
   echo "  -m, --api-admin-port int    api admin port (default: 5001)"
   echo "  -w, --web-port int          web port (default: 3000)"
-  echo "  -t, --tag string            image tag (default: ${VERSION})"
+  echo "  -t, --tag string            docker image tag (default: ${VERSION})"
+  echo "  --args string               docker arguments"
   echo
   title "FLAGS:"
   echo "  -b, --build           build images from source"
   echo "  -s, --seed            seed HTTP API server with metadata"
   echo "  -d, --detach          run in the background"
+  echo "  --no-web              don't start the web UI"
+  echo "  --no-volumes          don't create volumes"
   echo "  -h, --help            show help for script"
   exit 1
 }
@@ -50,13 +57,19 @@ usage() {
 project_root=$(git rev-parse --show-toplevel)
 cd "${project_root}/"
 
+# Base docker compose file
 compose_files="-f docker-compose.yml"
-args="-V --force-recreate --remove-orphans"
 
+# Default args
 API_PORT=5000
 API_ADMIN_PORT=5001
 WEB_PORT=3000
-TAG=${VERSION}
+NO_WEB="false"
+NO_VOLUMES="false"
+TAG="${VERSION}"
+BUILD="false"
+ARGS="-V --force-recreate --remove-orphans"
+# Parse args
 while [ $# -gt 0 ]; do
   case $1 in
     -a|'--api-port')
@@ -75,15 +88,20 @@ while [ $# -gt 0 ]; do
        shift
        TAG="${1}"
        ;;
+    --args)
+       shift
+       ARGS+=" ${1}"
+       ;;
     -b|'--build')
        BUILD='true'
+       TAG="${BUILD_VERSION}"
        ;;
     -s|'--seed')
        SEED='true'
        ;;
-    -d|'--detach')
-       DETACH='true'
-       ;;
+    -d|'--detach') DETACH='true' ;;
+    --no-web) NO_WEB='true' ;;
+    --no-volumes) NO_VOLUMES='true' ;;
     -h|'--help')
        usage
        exit 0
@@ -95,19 +113,34 @@ while [ $# -gt 0 ]; do
   shift
 done
 
+# Enable detach mode to run containers in background
 if [[ "${DETACH}" = "true" ]]; then
-  args+=" -d"
+  ARGS+=" --detach"
 fi
 
+# Enable starting HTTP API server with sample metadata
+if [[ "${SEED}" = "true" ]]; then
+  compose_files+=" -f docker-compose.seed.yml"
+fi
+
+# Enable building from source
 if [[ "${BUILD}" = "true" ]]; then
   compose_files+=" -f docker-compose.dev.yml"
-  args+=" --build"
+  ARGS+=" --build"
 fi
 
-if [[ "${SEED}" = "true" ]]; then
-  compose_files+=" -f docker-compose.seed.yml"
+# Enable web UI
+if [[ "${NO_WEB}" = "false" ]]; then
+  # Enable building web UI from source; otherwise use 'latest' build
+  [[ "${BUILD}" = "true" ]] && compose_files+=" -f docker-compose.web-dev.yml" \
+    || compose_files+=" -f docker-compose.web.yml"
 fi
 
-${DOCKER_DIR}/volumes.sh marquez
+# Create docker volumes for Marquez
+if [[ "${NO_VOLUMES}" = "false" ]]; then
+  ./docker/volumes.sh marquez
+fi
 
-API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG=${TAG} docker-compose $compose_files up $args
+# Run docker compose cmd with overrides
+DOCKER_SCAN_SUGGEST=false API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG=${TAG} \
+  docker-compose --log-level ERROR $compose_files up $ARGS