refactor: updates for api-db and keycloak-db to support MySQL8 (#3816)

.github/workflows/test-db-migrations.yml

@@ -47,7 +47,7 @@ jobs:
       -
         name: Show initial migration logs
         run: |
-          docker compose -p lagoon logs api-init
+          docker compose -p lagoon logs api-db-init
       -
         name: Initiate rollback
         run: |

Jenkinsfile

@@ -63,10 +63,13 @@ pipeline {
       }
       steps {
         sh script: "make -j$NPROC -O build", label: "Building images"
+        sh script: 'make go/test'
+        retry(3) {
+          sh script: "make -j$NPROC -O build PLATFORM_ARCH=linux/arm64", label: "Building arm images"
+        }
         retry(3) {
           sh script: 'docker login -u amazeeiojenkins -p $PASSWORD', label: "Docker login"
           sh script: "make -O publish-testlagoon-images PUBLISH_PLATFORM_ARCH=linux/amd64 BRANCH_NAME=${SAFEBRANCH_NAME}", label: "Publishing built amd64 images to testlagoon/*"
-          sh script: 'make go/test'
         }
       }
     }

Makefile

@@ -98,13 +98,21 @@ KUBECTL = $(realpath ./local-dev/kubectl)
 JQ = $(realpath ./local-dev/jq)
 K3D = $(realpath ./local-dev/k3d)
 
+# which database vendor type to use, can be mariadb (default) or mysql
+DATABASE_VENDOR = mariadb
+# DATABASE_VENDOR = mysql
+DATABASE_DOCKERFILE = Dockerfile
+ifeq ($(DATABASE_VENDOR), mysql)
+DATABASE_DOCKERFILE = Dockerfile.mysql
+endif
+
 #######
 ####### Functions
 #######
 
 # Builds a docker image. Expects as arguments: name of the image, location of Dockerfile, path of
 # Docker Build Context
-docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
+docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
 
 docker_buildx_create = 	docker buildx create --name $(CI_BUILD_TAG) || echo  -e '$(CI_BUILD_TAG) builder already present\n'
 
@@ -190,13 +198,13 @@ $(build-services):
 
 # Dependencies of Service Images
 build/auth-server build/webhook-handler build/webhooks2tasks build/api: build/yarn-workspace-builder
-build/api-db: services/api-db/Dockerfile
+build/api-db: services/api-db/$(DATABASE_DOCKERFILE)
 build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
 build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile
-build/keycloak-db: services/keycloak-db/Dockerfile
+build/keycloak-db: services/keycloak-db/$(DATABASE_DOCKERFILE)
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
 build/tests: tests/Dockerfile
@@ -309,14 +317,14 @@ webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test
 
 .PHONY: publish-testlagoon-images
 publish-testlagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 # tag and push all images
 
 .PHONY: publish-uselagoon-images
 publish-uselagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 .PHONY: clean
 clean:
@@ -723,6 +731,7 @@ endif
 		OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 		IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 		SKIP_INSTALL_REGISTRY=true \
+		CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 		LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 		USE_CALICO_CNI=false \
 		LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \
@@ -796,7 +805,6 @@ k3d/local-dev-logging:
 		&& echo -e 'with a default `container-logs-*` pattern'
 
 
-
 # k3d/push-images pushes locally build images into the k3d cluster registry.
 IMAGES = $(K3D_SERVICES) $(LOCAL_DEV_SERVICES) $(TASK_IMAGES)
 .PHONY: k3d/push-images
@@ -930,6 +938,7 @@ k3d/retest:
 			OVERRIDE_ACTIVE_STANDBY_TASK_IMAGE="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library/task-activestandby:$(SAFE_BRANCH_NAME)" \
 			IMAGE_REGISTRY="registry.$$($(KUBECTL) -n ingress-nginx get services ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}').nip.io/library" \
 			SKIP_ALL_DEPS=true \
+			CORE_DATABASE_VENDOR=$(DATABASE_VENDOR) \
 			LAGOON_FEATURE_FLAG_DEFAULT_ISOLATION_NETWORK_POLICY=enabled \
 			USE_CALICO_CNI=false \
 			LAGOON_SSH_PORTAL_LOADBALANCER=$(LAGOON_SSH_PORTAL_LOADBALANCER) \

docker-bake.hcl

@@ -15,6 +15,14 @@ variable "UPSTREAM_REPO" {
   default = "uselagoon"
 }
 
+variable "DATABASE_DOCKERFILE" {
+  default = "Dockerfile"
+}
+
+variable "DATABASE_VENDOR" {
+  default = "mariadb"
+}
+
 variable "UPSTREAM_TAG" {
   default = "latest"
 }
@@ -39,6 +47,7 @@ target "default"{
     LAGOON_VERSION = "${LAGOON_VERSION}"
     UPSTREAM_REPO = "${UPSTREAM_REPO}"
     UPSTREAM_TAG = "${UPSTREAM_TAG}"
+    DATABASE_VENDOR = "${DATABASE_VENDOR}"
   }
 }
 
@@ -66,6 +75,40 @@ group "default" {
   ]
 }
 
+group "go-services" {
+  targets = [
+    "actions-handler",
+    "backup-handler",
+    "api-sidecar-handler",
+    "logs2notifications",
+    "task-activestandby",
+    "workflows"
+  ]
+}
+
+group "js-services" {
+  targets = [
+    "api",
+    "auth-server",
+    "webhook-handler",
+    "webhooks2tasks",
+  ]
+}
+
+group "other" {
+  targets = [
+    "api-db",
+    "api-redis",
+    "broker",
+    "keycloak-db",
+    "keycloak",
+    "ssh",
+    "local-api-data-watcher-pusher",
+    "local-git",
+    "tests",
+  ]
+}
+
 group "ui-logs-development" {
   targets = [
     "actions-handler",
@@ -131,6 +174,7 @@ target "api" {
 target "api-db" {
   inherits = ["default"]
   context = "services/api-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/api-db - the MariaDB database service for Lagoon API"
   }
@@ -206,6 +250,7 @@ target "keycloak" {
 target "keycloak-db" {
   inherits = ["default"]
   context = "services/keycloak-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/keycloak-db - the MariaDB database service for Lagoon Keycloak"
   }

docker-compose.local-dev.yaml

@@ -29,6 +29,7 @@ services:
   api:
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
     volumes:
+      - ./services/api/database:/app/services/api/database
       - ./services/api/src:/app/services/api/src
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist

services/api-db/Dockerfile

@@ -12,6 +12,9 @@ COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
 RUN chown -R mysql /legacy-migration-scripts/ \
     && /bin/fix-permissions /legacy-migration-scripts/
 
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
+
 USER mysql
 
 ENV MARIADB_DATABASE=infrastructure \

services/api-db/Dockerfile.mysql

@@ -0,0 +1,30 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+USER root
+
+COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
+RUN chown -R mysql /legacy-migration-scripts/ \
+    && /bin/fix-permissions /legacy-migration-scripts/
+
+# replace the generate-env script with our password, since generate-env only adds a domain which isnt valid in this image
+COPY password-entrypoint.bash /lagoon/entrypoints/55-generate-env.sh
+
+USER mysql
+
+ENV MYSQL_DATABASE=infrastructure \
+    MYSQL_USER=api \
+    MYSQL_PASSWORD=api
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# do we need to keep these any more? v2.10.0 was long ago...
+# COPY ./rerun_initdb.sh /rerun_initdb.sh
+# COPY ./legacy_rerun_initdb.sh /legacy_rerun_initdb.sh
+
+CMD ["mysqld", "--sql_mode", ""]

services/api-db/password-entrypoint.bash

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ ${API_DB_PASSWORD+x} ]; then
+    if [ "${LAGOON}" == "mysql" ]; then
+        export MYSQL_PASSWORD=${API_DB_PASSWORD}
+    else
+        export MARIADB_PASSWORD=${API_DB_PASSWORD}
+    fi
+fi

services/api/database/migrations/20220908065119_initial_db.js

@@ -9,18 +9,18 @@
             .createTable('advanced_task_definition', function (table) {
                 table.increments('id').notNullable().primary();
                 table.string('name', 300).notNullable();
-                table.specificType('description', 'text').defaultTo('').notNullable();
+                table.specificType('description', 'text').notNullable();
                 table.string('image', 2000).defaultTo('');
                 table.string('service', 100);
                 table.string('type', 100).notNullable();
                 table.integer('environment');
                 table.integer('project');
-                table.string('group_name', 2000);
+                table.string('group_name', 300);
                 table.enu('permission', ['GUEST', 'DEVELOPER', 'MAINTAINER']).defaultTo('GUEST');
-                table.specificType('command', 'text').defaultTo('');
+                table.specificType('command', 'text');
                 table.string('confirmation_text', 2000);
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['name', 'environment', 'project', 'group_name'], {indexName: 'name', storageEngineIndexType: 'hash'});
             })
             .createTable('advanced_task_definition_argument', function (table) {
@@ -76,8 +76,8 @@
                 table.string('name', 100);
                 table.integer('project');
                 table.enu('deploy_type', ['branch', 'pullrequest', 'promote']).notNullable();
-                table.string('deploy_base_ref', 100);
-                table.string('deploy_head_ref', 100);
+                table.string('deploy_base_ref', 250);
+                table.string('deploy_head_ref', 250);
                 table.string('deploy_title', 300);
                 table.enu('environment_type', ['production', 'development']).notNullable();
                 table.integer('auto_idle', 1).notNullable().defaultTo(1);
@@ -89,7 +89,7 @@
                 table.string('openshift_project_pattern', 300);
                 table.timestamp('updated').notNullable().defaultTo(knex.fn.now());
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['project', 'name', 'deleted'], {indexName: 'project_name_deleted'});
             })
             .createTable('environment_backup', function (table) {
@@ -98,7 +98,7 @@
                 table.string('source', 300);
                 table.string('backup_id', 300).unique({indexName:'backup_id'});
                 table.timestamp('created').notNullable().defaultTo(knex.raw('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'));
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.index('environment', 'backup_environment');
             })
             .createTable('environment_fact', function (table) {
@@ -109,9 +109,9 @@
                 table.string('value', 300).notNullable();
                 table.enu('type', ['TEXT', 'URL', 'SEMVER']).defaultTo('TEXT');
                 table.string('source', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
+                table.specificType('description', 'text');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.specificType('category', 'text').defaultTo('');
+                table.specificType('category', 'text');
                 table.boolean('key_fact').notNullable().defaultTo(0);
                 table.unique(['environment', 'name', 'source'], {indexName: 'environment_fact'});
             })
@@ -126,17 +126,17 @@
                 table.integer('environment');
                 table.string('severity', 300).defaultTo('');
                 table.decimal('severity_score', 1, 1).defaultTo(0);
-                table.string('identifier', 300).notNullable();
-                table.string('lagoon_service', 300).defaultTo('');
+                table.string('identifier', 100).notNullable();
+                table.string('lagoon_service', 100).defaultTo('');
                 table.string('source', 300).defaultTo('');
                 table.string('associated_package', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
-                table.string('version', 300).defaultTo('');
+                table.specificType('description', 'text');
+                table.string('version', 100).defaultTo('');
                 table.string('fixed_version', 300).defaultTo('');
                 table.string('links', 300).defaultTo('');
                 table.json('data');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['environment', 'lagoon_service', 'version', 'identifier', 'deleted'], {indexName: 'environment'});
             })
             .createTable('environment_service', function (table) {
@@ -235,7 +235,7 @@
                 table.integer('development_environments_limit');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
                 table.string('private_key', 5000);
-                table.json('metadata').defaultTo('{}');
+                table.json('metadata');
                 table.string('router_pattern', 300).defaultTo(null);
             })
             .createTable('project_notification', function (table) {
@@ -251,7 +251,7 @@
                 table.string('filename', 100).notNullable();
                 table.text('s3_key').notNullable();
                 table.datetime('created').notNullable().defaultTo(knex.fn.now());
-                table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.datetime('deleted').notNullable();
             })
             .createTable('ssh_key', function (table) {
                table.increments('id').notNullable().primary();
@@ -295,7 +295,7 @@
                 table.integer('project', 11).notNullable();
                 table.integer('advanced_task_definition', 11).notNullable();
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
             })
         }
         else {

services/api/database/migrations/20230801000000_organizations.js

@@ -10,7 +10,7 @@ exports.up = async function(knex) {
             table.increments('id').notNullable().primary();
             table.string('name', 300).unique({indexName: 'name'});
             table.string('friendly_name', 300).notNullable();
-            table.specificType('description', 'text').defaultTo('').notNullable();
+            table.specificType('description', 'text').notNullable();
             table.integer('quota_project').defaultTo(1).notNullable();
             table.integer('quota_group').defaultTo(10).notNullable();
             table.integer('quota_notification').defaultTo(10).notNullable();

services/api/database/migrations/20250131000000_source_type_enums.js

@@ -0,0 +1,22 @@
+exports.up = function(knex) {
+    // fix the way the enum values are stored to be lowercase for mysql strict
+    return Promise.all([
+        knex('deployment')
+            .where('source_type', '=', 'API')
+            .update('source_type', 'api'),
+        knex('deployment')
+            .where('source_type', '=', 'WEBHOOK')
+            .update('source_type', 'webhook'),
+        knex('task')
+            .where('source_type', '=', 'API')
+            .update('source_type', 'api'),
+    ]);
+  };
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function(knex) {
+return knex.schema;
+};