Validate the configuration as part of a pre-flight check

.werft/jobs/build/build-and-publish.ts

@@ -154,8 +154,8 @@ function publishKots(werft: Werft, jobConfig: JobConfig) {
         { slice: phases.PUBLISH_KOTS },
     );
 
-    // Generate the logo and pull any Helm charts
-    exec(`make logo helm -C ${REPLICATED_DIR}`, { slice: phases.PUBLISH_KOTS });
+    // Generate the preflights, logo and pull any Helm charts
+    exec(`make generate_preflight_checks logo helm -C ${REPLICATED_DIR}`, { slice: phases.PUBLISH_KOTS });
 
     // Update the additionalImages in the kots-app.yaml
     exec(`/tmp/installer mirror kots --file ${REPLICATED_YAML_DIR}/kots-app.yaml`, { slice: phases.PUBLISH_KOTS });

install/installer/leeway.Dockerfile

@@ -4,7 +4,7 @@
 
 FROM alpine:3.16
 COPY --from=alpine/helm:3.8.0 /usr/bin/helm /usr/bin/helm
-RUN apk add --no-cache curl jq openssh-keygen yq  \
+RUN apk add --no-cache bash curl jq openssh-keygen yq  \
     && curl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl \
     && chmod +x /usr/local/bin/kubectl
 COPY install-installer--app/installer install-installer--app/provenance-bundle.jsonl /app/

install/installer/scripts/kots-install.sh

@@ -1,143 +1,177 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright (c) 2022 Gitpod GmbH. All rights reserved.
 # Licensed under the MIT License. See License-MIT.txt in the project root for license information.
 
-set -e
+set -eo pipefail
 
-echo "Gitpod: Killing any in-progress installations"
+INSTALLER_LOG_FILE=/tmp/gitpod-installer.log
+rm -f "${INSTALLER_LOG_FILE}"
 
-kubectl delete jobs.batch -n "${NAMESPACE}" -l component="gitpod-installer,cursor!=${CURSOR}" --force --grace-period 0 || true
-kubectl delete pod -n "${NAMESPACE}" -l component="gitpod-installer,cursor!=${CURSOR}" --force --grace-period 0 || true
+trap 'catch $?' EXIT
 
-if [ "$(helm status -n "${NAMESPACE}" gitpod -o json | jq '.info.status == "deployed"')" = "false" ];
-then
-    echo "Gitpod: Deployment in-progress - clearing"
+catch() {
+    echo "Gitpod: Saving log to configmap"
 
-    VERSION="$(helm status -n "${NAMESPACE}" gitpod -o json | jq '.version')"
-    if [ "${VERSION}" -le 1 ];
-    then
-        echo "Gitpod: Uninstall application"
-        helm uninstall -n "${NAMESPACE}" gitpod --wait || true
+    # Allow config map to be updated
+    kubectl create configmap \
+        gitpod-installation-status \
+        -n "${NAMESPACE}" \
+        --from-file="${INSTALLER_LOG_FILE}" \
+        -o yaml \
+        --dry-run=client \
+        | kubectl apply -f -
+
+    # This line is used by KOTS to analyse the validation status
+    if [ "${1}" = "0" ]; then
+        echo "Gitpod: status pass"
     else
-        echo "Gitpod: Rolling back application"
-        helm rollback -n "${NAMESPACE}" gitpod --wait || true
+        echo "Gitpod: status fail"
+    fi
+    exit "${1}"
+}
+
+main() {
+    if [ "${INSTALLER_DRY_RUN}" != "true" ]; then
+        echo "Gitpod: Killing any in-progress installations"
+
+        kubectl delete jobs.batch -n "${NAMESPACE}" -l component="gitpod-installer,cursor!=${CURSOR}" --force --grace-period 0 || true
+        kubectl delete pod -n "${NAMESPACE}" -l component="gitpod-installer,cursor!=${CURSOR}" --force --grace-period 0 || true
+
+        if [ "$(helm status -n "${NAMESPACE}" gitpod -o json | jq '.info.status == "deployed"')" = "false" ];
+        then
+            echo "Gitpod: Deployment in-progress - clearing"
+
+            VERSION="$(helm status -n "${NAMESPACE}" gitpod -o json | jq '.version')"
+            if [ "${VERSION}" -le 1 ];
+            then
+                echo "Gitpod: Uninstall application"
+                helm uninstall -n "${NAMESPACE}" gitpod --wait || true
+            else
+                echo "Gitpod: Rolling back application"
+                helm rollback -n "${NAMESPACE}" gitpod --wait || true
+            fi
+        fi
     fi
-fi
 
-echo "Gitpod: Create a Helm template directory"
-rm -Rf "${GITPOD_OBJECTS}"
-mkdir -p "${GITPOD_OBJECTS}/templates"
-cat <<EOF >> "${GITPOD_OBJECTS}/Chart.yaml"
+    echo "Gitpod: Create a Helm template directory"
+    rm -Rf "${GITPOD_OBJECTS}"
+    mkdir -p "${GITPOD_OBJECTS}/templates"
+    cat <<EOF >> "${GITPOD_OBJECTS}/Chart.yaml"
 apiVersion: v2
 name: gitpod-kots
 description: Always ready-to-code
 version: "1.0.0"
 appVersion: "$(/app/installer version | yq e '.version' -)"
 EOF
 
-echo "Gitpod: Generate the base Installer config"
-/app/installer config init
+    echo "Gitpod: Generate the base Installer config"
+    /app/installer config init
 
-echo "Gitpod: auto-detecting ShiftFS support on host machine"
-/app/installer config cluster shiftfs
+    if [ "${INSTALLER_DRY_RUN}" != "true" ]; then
+        echo "Gitpod: auto-detecting ShiftFS support on host machine"
+        /app/installer config cluster shiftfs
+    fi
 
-echo "Gitpod: auto-detecting containerd settings on host machine"
-/app/installer config files containerd
+    echo "Gitpod: auto-detecting containerd settings on host machine"
+    /app/installer config files containerd
 
-echo "Gitpod: auto-detecting settings"
-/app/installer config build-from-envvars
+    echo "Gitpod: auto-detecting settings"
+    /app/installer config build-from-envvars
 
-echo "Gitpod: Validate config"
-/app/installer validate config
+    echo "Gitpod: Validate config"
+    /app/installer validate config
 
-echo "Gitpod: render Kubernetes manifests"
-/app/installer render --use-experimental-config > "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+    echo "Gitpod: render Kubernetes manifests"
+    /app/installer render --use-experimental-config > "${GITPOD_OBJECTS}/templates/gitpod.yaml"
 
-if [ "${INSTALLER_DRY_RUN}" = "true" ]; then
-    echo "Gitpod: dry-run set to true, no installation will be performed"
-    exit
-fi
+    if [ "${INSTALLER_DRY_RUN}" = "true" ]; then
+        echo "Gitpod: dry-run set to true, no installation will be performed"
+        return 0
+    fi
 
-# Combine the pull secrets
-echo "${LOCAL_REGISTRY_IMAGE_PULL_DOCKER_CONFIG_JSON}" > /tmp/kotsregistry.json
-if [ "${REG_INCLUSTER_ENABLED}" = "1" ]; then
-    echo "Gitpod: Add the local registry secret to the in-cluster registry secret"
+    # Combine the pull secrets
+    echo "${LOCAL_REGISTRY_IMAGE_PULL_DOCKER_CONFIG_JSON}" > /tmp/kotsregistry.json
+    if [ "${REG_INCLUSTER_ENABLED}" = "1" ]; then
+        echo "Gitpod: Add the local registry secret to the in-cluster registry secret"
 
-    # Get the in-cluster registry secret
-    yq eval-all '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson")' \
-        "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
-        | base64 -d \
-        > /tmp/gitpodregistry.json
+        # Get the in-cluster registry secret
+        yq eval-all '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson")' \
+            "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
+            | base64 -d \
+            > /tmp/gitpodregistry.json
 
-    REGISTRY_SECRET="$(cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - | base64 -w 0)"
-    export REGISTRY_SECRET
+        REGISTRY_SECRET="$(cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - | base64 -w 0)"
+        export REGISTRY_SECRET
 
-    echo "Gitpod: update the in-cluster registry secret"
-    yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
-        "${GITPOD_OBJECTS}/templates/gitpod.yaml"
-else
-    echo "Gitpod: configuring external container registry"
+        echo "Gitpod: update the in-cluster registry secret"
+        yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == "builtin-registry-auth") | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
+            "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+    else
+        echo "Gitpod: configuring external container registry"
 
-    # Get the external-container-registry secret so we can merge the external registry and KOTS registry keys
-    echo "${EXTERNAL_DOCKER_CONFIG_JSON}" > /tmp/gitpodregistry.json
+        # Get the external-container-registry secret so we can merge the external registry and KOTS registry keys
+        echo "${EXTERNAL_DOCKER_CONFIG_JSON}" > /tmp/gitpodregistry.json
 
-    cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - > /tmp/container-registry-secret
+        cat /tmp/kotsregistry.json /tmp/gitpodregistry.json | jq -s '.[0] * .[1]' - - > /tmp/container-registry-secret
 
-    echo "Gitpod: append the container-registry secret"
-    echo "---" >> "${GITPOD_OBJECTS}/templates/gitpod.yaml"
-    kubectl create secret docker-registry "${REG_EXTERNAL_CERTIFICATE_NAME}" \
-        --namespace "${NAMESPACE}" \
-        --from-file=.dockerconfigjson=/tmp/container-registry-secret \
-        -o yaml --dry-run=client >> "${GITPOD_OBJECTS}/templates/gitpod.yaml"
-fi
+        echo "Gitpod: append the container-registry secret"
+        echo "---" >> "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+        kubectl create secret docker-registry "${REG_EXTERNAL_CERTIFICATE_NAME}" \
+            --namespace "${NAMESPACE}" \
+            --from-file=.dockerconfigjson=/tmp/container-registry-secret \
+            -o yaml --dry-run=client >> "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+    fi
 
-if [ "${REG_DOCKER_CONFIG_ENABLED}" = "1" ];
-then
-    # Work out the registry secret to use
-    if [ "${REG_INCLUSTER_ENABLED}" = "0" ];
+    if [ "${REG_DOCKER_CONFIG_ENABLED}" = "1" ];
     then
-        export REGISTRY_SECRET_NAME="${REG_EXTERNAL_CERTIFICATE_NAME}"
-    else
-        export REGISTRY_SECRET_NAME="builtin-registry-auth"
+        # Work out the registry secret to use
+        if [ "${REG_INCLUSTER_ENABLED}" = "0" ];
+        then
+            export REGISTRY_SECRET_NAME="${REG_EXTERNAL_CERTIFICATE_NAME}"
+        else
+            export REGISTRY_SECRET_NAME="builtin-registry-auth"
+        fi
+
+        echo "Gitpod: Add given extra docker config JSON file to ${REGISTRY_SECRET_NAME}"
+        yq eval-all '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson")' \
+            "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
+            | base64 -d \
+            > /tmp/currentconfig.json
+
+        echo "Gitpod: update the in-cluster registry secret"
+        REGISTRY_SECRET="$(jq -s '.[0] * .[1]' /tmp/userconfig.json /tmp/currentconfig.json | base64 -w 0)"
+        export REGISTRY_SECRET
+        yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
+            "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+    fi
+
+    echo "Gitpod: Escape any Golang template values"
+    # shellcheck disable=SC2016
+    sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml"
+
+    # If certificate secret already exists, set the timeout to 5m
+    CERT_SECRET=$(kubectl get secrets -n "${NAMESPACE}" https-certificates -o jsonpath='{.metadata.name}' || echo '')
+    HELM_TIMEOUT="5m"
+    if [ "${CERT_SECRET}" = "" ]; then
+        HELM_TIMEOUT="1h"
     fi
 
-    echo "Gitpod: Add given extra docker config JSON file to ${REGISTRY_SECRET_NAME}"
-    yq eval-all '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson")' \
-        "${GITPOD_OBJECTS}/templates/gitpod.yaml" \
-        | base64 -d \
-        > /tmp/currentconfig.json
-
-    echo "Gitpod: update the in-cluster registry secret"
-    REGISTRY_SECRET="$(jq -s '.[0] * .[1]' /tmp/userconfig.json /tmp/currentconfig.json | base64 -w 0)"
-    export REGISTRY_SECRET
-    yq eval-all --inplace '(select(.kind == "Secret" and .metadata.name == env(REGISTRY_SECRET_NAME)) | .data.".dockerconfigjson") |= env(REGISTRY_SECRET)' \
-        "${GITPOD_OBJECTS}/templates/gitpod.yaml"
-fi
-
-echo "Gitpod: Escape any Golang template values"
-# shellcheck disable=SC2016
-sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml"
-
-# If certificate secret already exists, set the timeout to 5m
-CERT_SECRET=$(kubectl get secrets -n "${NAMESPACE}" https-certificates -o jsonpath='{.metadata.name}' || echo '')
-HELM_TIMEOUT="5m"
-if [ "${CERT_SECRET}" = "" ]; then
-    HELM_TIMEOUT="1h"
-fi
-
-# The long timeout is to ensure the TLS cert is created (if required)
-echo "Gitpod: Apply the Kubernetes objects with timeout of ${HELM_TIMEOUT}"
-helm upgrade \
-    --atomic \
-    --cleanup-on-fail \
-    --create-namespace \
-    --install \
-    --namespace="${NAMESPACE}" \
-    --reset-values \
-    --timeout "${HELM_TIMEOUT}" \
-    --wait \
-    gitpod \
-    "${GITPOD_OBJECTS}"
-
-echo "Gitpod: Restarting installation status job"
-kubectl delete pod -n "${NAMESPACE}" -l component=gitpod-installer-status || true
+    # The long timeout is to ensure the TLS cert is created (if required)
+    echo "Gitpod: Apply the Kubernetes objects with timeout of ${HELM_TIMEOUT}"
+    helm upgrade \
+        --atomic \
+        --cleanup-on-fail \
+        --create-namespace \
+        --install \
+        --namespace="${NAMESPACE}" \
+        --reset-values \
+        --timeout "${HELM_TIMEOUT}" \
+        --wait \
+        gitpod \
+        "${GITPOD_OBJECTS}"
+
+    echo "Gitpod: Restarting installation status job"
+    kubectl delete pod -n "${NAMESPACE}" -l component=gitpod-installer-status || true
+}
+
+main 2>&1 | tee -a "${INSTALLER_LOG_FILE}"

install/kots/Makefile

@@ -4,7 +4,7 @@ CHANNEL_UNSTABLE = Unstable
 CHARTS_DIR = charts
 YAML_DIR = manifests
 
-all: logo helm lint create_dev_release
+all: generate_preflight_checks logo helm lint create_dev_release
 
 create_dev_release:
 	@if [ "${REPLICATED_DEV_CHANNEL}" = "" ]; then \
@@ -24,6 +24,31 @@ create_unstable_release:
 	replicated release create --lint --ensure-channel --yaml-dir ${YAML_DIR} --promote ${CHANNEL_UNSTABLE}
 .PHONY: create_unstable_release
 
+generate_preflight_checks:
+	@echo "Generating installation config validation preflight check"
+
+# Extract the installer job
+	@yq r manifests/gitpod-installer-job.yaml spec.template.spec > /tmp/installer-job.yaml
+# Remove the envFrom block
+	@yq d -i /tmp/installer-job.yaml containers[0].envFrom
+
+# Extract the envFrom block as envvars and convert to "env" format
+	@yq r -j manifests/gitpod-kots-config.yaml data > /tmp/installer-envvars.json
+
+	@for row in $$(cat /tmp/installer-envvars.json | jq -r 'to_entries | .[] | @base64'); do \
+		yq w -i /tmp/installer-job.yaml containers[0].env[+].name "$$(echo $${row} | base64 -d | jq -r '.key')"; \
+		yq w -i /tmp/installer-job.yaml containers[0].env[-1].value "$$(echo $${row} | base64 -d | jq -r '.value')"; \
+	done
+
+# Set as dry run
+	@yq w -i /tmp/installer-job.yaml containers[0].env[+].name "INSTALLER_DRY_RUN"
+	@yq w --tag=!!str -i /tmp/installer-job.yaml containers[0].env[-1].value "true"
+
+# Merge the envvars into the installer job
+	@yq p -i /tmp/installer-job.yaml spec.collectors[0].runPod.podSpec
+	@yq m -i manifests/kots-preflight.yaml /tmp/installer-job.yaml
+.PHONY: generate_preflight_checks
+
 helm:
 	@echo "Installing Helm dependencies"
 	@rm -f ${YAML_DIR}/*.tgz

install/kots/manifests/gitpod-installation-status.yaml

@@ -26,11 +26,11 @@ spec:
         component: gitpod-installer-status
     spec:
       restartPolicy: Always
-      serviceAccountName: installer
+      serviceAccountName: kotsadm
       containers:
         - name: installation-status
           # This will normally be the release tag
-          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-kots-refactoring.6"
+          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-kots-config-validate.7"
           envFrom:
             - configMapRef:
                 name: gitpod-kots-config

install/kots/manifests/gitpod-installer-job.yaml

@@ -34,12 +34,12 @@ spec:
               - matchExpressions:
                 - key: gitpod.io/workload_workspace_headless
                   operator: Exists
-      serviceAccountName: installer
+      serviceAccountName: kotsadm
       restartPolicy: OnFailure
       containers:
         - name: installer
           # This will normally be the release tag
-          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-kots-refactoring.6"
+          image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-kots-config-validate.7"
           volumeMounts:
             - mountPath: /mnt/node0
               name: node-fs0