[werft] Integrate Installer with Deploy

The Installer is the default for new Preview Environments. Preview Environments powered by Helm can switch to the Installer if 'with-clean-slate-deployment=true' is specified. Folks can continue using Helm if 'with-helm=true' is specified. Co-authored-by: Christian Weichel <chris@gitpod.io>
gitpod-io · Dec 13, 2021 · 7ac89a8 · 7ac89a8
1 parent b08041e
commit 7ac89a8
Show file tree

Hide file tree

Showing 13 changed files with 763 additions and 74 deletions.
diff --git a/.werft/build.ts b/.werft/build.ts
diff --git a/.werft/post-process.sh b/.werft/post-process.sh
diff --git a/.werft/util/kubectl.ts b/.werft/util/kubectl.ts
@@ -5,29 +5,53 @@ import { getGlobalWerftInstance } from './werft';
 
 export const IS_PREVIEW_APP_LABEL: string = "isPreviewApp";
 
+export const helmInstallName = "gitpod";
+
 export function setKubectlContextNamespace(namespace: string, shellOpts: ExecOptions) {
     [
         `kubectl config current-context`,
         `kubectl config set-context --current --namespace=${namespace}`
     ].forEach(cmd => exec(cmd, shellOpts));
 }
 
+export async function wipePreviewEnvironmentAndNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+    // wipe preview envs built with installer
+    await wipePreviewEnvironmentInstaller(namespace, shellOpts);
+
+    // wipe preview envs previously built with helm
+    await wipePreviewEnvironmentHelm(helmInstallName, namespace, shellOpts)
+
+    deleteAllWorkspaces(namespace, shellOpts);
+
+    await deleteAllUnnamespacedObjects(namespace, shellOpts);
+
+    deleteNamespace(true, namespace, shellOpts);
+}
+
 export async function wipeAndRecreateNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
-    await wipePreviewEnvironment(helmInstallName, namespace, shellOpts);
+    await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, shellOpts);
 
     createNamespace(namespace, shellOpts);
 }
 
-export async function wipePreviewEnvironment(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+export async function wipePreviewEnvironmentHelm(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
     // uninstall helm first so that:
     //  - ws-scaler can't create new ghosts in the meantime
     //  - ws-manager can't start new probes/workspaces
     uninstallHelm(helmInstallName, namespace, shellOpts)
+}
 
-    deleteAllWorkspaces(namespace, shellOpts);
-    await deleteAllUnnamespacedObjects(namespace, shellOpts);
+async function wipePreviewEnvironmentInstaller(namespace: string, shellOpts: ExecOptions) {
+    const slice = shellOpts.slice || "installer";
+    const werft = getGlobalWerftInstance();
 
-    deleteNamespace(true, namespace, shellOpts);
+    const hasGitpodConfigmap = (exec(`kubectl -n ${namespace} get configmap gitpod-app`, { slice, dontCheckRc: true })).code === 0;
+    if (hasGitpodConfigmap) {
+        werft.log(slice, `${namespace} has Gitpod configmap, proceeding with removal`);
+        exec(`./.werft/util/uninstall-gitpod.sh ${namespace}`, { slice });
+    } else {
+        werft.log(slice, `There is no Gitpod configmap, moving on`);
+    }
 }
 
 function uninstallHelm(installationName: string, namespace: string, shellOpts: ExecOptions) {
@@ -43,6 +67,7 @@ function uninstallHelm(installationName: string, namespace: string, shellOpts: E
     exec(`helm --namespace ${namespace} delete ${installationName} --wait`, shellOpts);
 }
 
+// Delete pods for running workspaces, even if they are stuck in terminating because of the finalizer decorator
 function deleteAllWorkspaces(namespace: string, shellOpts: ExecOptions) {
     const objs = exec(`kubectl get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, async: false })
         .split("\n")
@@ -74,13 +99,14 @@ async function deleteAllUnnamespacedObjects(namespace: string, shellOpts: ExecOp
 
     const promisedDeletes: Promise<any>[] = [];
     for (const resType of ["clusterrole", "clusterrolebinding", "podsecuritypolicy"]) {
-        werft.log(slice, `Deleting old ${resType}s...`);
+        werft.log(slice, `Searching and filtering ${resType}s...`);
         const objs = exec(`kubectl get ${resType} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, slice, async: false })
             .split("\n")
             .map(o => o.trim())
             .filter(o => o.length > 0)
             .filter(o => o.startsWith(`${namespace}-ns-`)); // "{{ .Release.Namespace }}-ns-" is the prefix-pattern we use throughout our helm resources for un-namespaced resources
 
+        werft.log(slice, `Deleting old ${resType}s...`);
         for (const obj of objs) {
             promisedDeletes.push(exec(`kubectl delete ${resType} ${obj}`, { ...shellOpts, slice, async: true }) as Promise<any>);
         }
@@ -127,7 +153,7 @@ export function deleteNamespace(wait: boolean, namespace: string, shellOpts: Exe
     }
 }
 
-export function deleteNonNamespaceObjects(namespace: string, destname: string, shellOpts: ExecOptions) {
+export async function deleteNonNamespaceObjects(namespace: string, destname: string, shellOpts: ExecOptions) {
     exec(`/usr/local/bin/helm3 delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, { ...shellOpts });
 
     let objs = [];
@@ -141,9 +167,11 @@ export function deleteNonNamespaceObjects(namespace: string, destname: string, s
         )
     )
 
+    const promisedDeletes: Promise<any>[] = [];
     objs.forEach(o => {
-        exec(`kubectl delete ${o.kind} ${o.obj}`, shellOpts);
+        promisedDeletes.push(exec(`kubectl delete ${o.kind} ${o.obj}`, {...shellOpts, async: true}) as Promise<any>);
     });
+    await Promise.all(promisedDeletes);
 }
 
 export interface PortRange {

diff --git a/.werft/util/uninstall-gitpod.sh b/.werft/util/uninstall-gitpod.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+NAMESPACE=$1
+
+if [[ -z ${NAMESPACE} ]]; then
+   echo "One or more input params were invalid. The params we received were: ${NAMESPACE}"
+   exit 1
+fi
+
+echo "Removing Gitpod in namespace ${NAMESPACE}"
+kubectl get configmap gitpod-app -n "${NAMESPACE}" -o jsonpath='{.data.app\.yaml}' | kubectl delete -f -
+
+echo "Removing Gitpod storage from ${NAMESPACE}"
+kubectl -n "${NAMESPACE}" delete pvc data-mysql-0
+# the installer includes the minio PVC in it's config mpap, this is a "just in case"
+kubectl -n "${NAMESPACE}" delete pvc minio || true
+
+echo "Successfully removed Gitpod from ${NAMESPACE}"
diff --git a/.werft/wipe-devstaging.ts b/.werft/wipe-devstaging.ts
@@ -1,5 +1,5 @@
 import { Werft } from './util/werft'
-import { wipePreviewEnvironment, listAllPreviewNamespaces } from './util/kubectl';
+import { wipePreviewEnvironmentAndNamespace, listAllPreviewNamespaces, helmInstallName } from './util/kubectl';
 import * as fs from 'fs';
 import { deleteExternalIp } from './util/gcloud';
 import * as Tracing from './observability/tracing'
@@ -23,7 +23,7 @@ async function wipePreviewCluster(shellOpts: ExecOptions) {
     }
 
     for (const namespace of namespaces) {
-        await wipePreviewEnvironment("gitpod", namespace, { ...shellOpts, slice: 'wipe' });
+        await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, { ...shellOpts, slice: 'wipe' });
     }
 }
 

diff --git a/installer/pkg/components/cluster/certmanager.go b/installer/pkg/components/cluster/certmanager.go
@@ -6,6 +6,7 @@ package cluster
 
 import (
 	"fmt"
+
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	v1 "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
 	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
@@ -22,8 +23,9 @@ func certmanager(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&v1.Issuer{
 			TypeMeta: common.TypeMetaCertificateIssuer,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:   caIssuer,
-				Labels: common.DefaultLabels(Component),
+				Name:      caIssuer,
+				Labels:    common.DefaultLabels(Component),
+				Namespace: ctx.Namespace,
 			},
 			Spec: v1.IssuerSpec{IssuerConfig: v1.IssuerConfig{
 				SelfSigned: &v1.SelfSignedIssuer{},

diff --git a/installer/pkg/components/image-builder-mk3/clusterrole.go b/installer/pkg/components/image-builder-mk3/clusterrole.go
@@ -18,9 +18,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 	return []runtime.Object{&rbacv1.ClusterRole{
 		TypeMeta: common.TypeMetaClusterRole,
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-			Namespace: ctx.Namespace,
-			Labels:    common.DefaultLabels(Component),
+			Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+			Labels: common.DefaultLabels(Component),
 		},
 		Rules: []rbacv1.PolicyRule{{
 			APIGroups:     []string{"policy"},

diff --git a/installer/pkg/components/registry-facade/clusterrole.go b/installer/pkg/components/registry-facade/clusterrole.go
@@ -19,9 +19,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&rbacv1.ClusterRole{
 			TypeMeta: common.TypeMetaClusterRole,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-				Namespace: ctx.Namespace,
-				Labels:    common.DefaultLabels(Component),
+				Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+				Labels: common.DefaultLabels(Component),
 			},
 			Rules: []rbacv1.PolicyRule{{
 				APIGroups:     []string{"policy"},

diff --git a/installer/pkg/components/registry-facade/podsecuritypolicy.go b/installer/pkg/components/registry-facade/podsecuritypolicy.go
@@ -6,6 +6,7 @@ package registryfacade
 
 import (
 	"fmt"
+
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 
 	"k8s.io/api/policy/v1beta1"
@@ -17,9 +18,8 @@ func podsecuritypolicy(ctx *common.RenderContext) ([]runtime.Object, error) {
 	return []runtime.Object{&v1beta1.PodSecurityPolicy{
 		TypeMeta: common.TypeMetaPodSecurityPolicy,
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-			Namespace: ctx.Namespace,
-			Labels:    common.DefaultLabels(Component),
+			Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+			Labels: common.DefaultLabels(Component),
 			Annotations: map[string]string{
 				"seccomp.security.alpha.kubernetes.io/allowedProfileNames": "runtime/default",
 				"apparmor.security.beta.kubernetes.io/allowedProfileNames": "runtime/default",

diff --git a/installer/pkg/components/server/configmap.go b/installer/pkg/components/server/configmap.go
@@ -7,6 +7,7 @@ package server
 import (
 	"encoding/json"
 	"fmt"
+
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	"github.com/gitpod-io/gitpod/installer/pkg/components/workspace"
 	corev1 "k8s.io/api/core/v1"
@@ -54,7 +55,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 			MinAgeDays:                 14,
 			MinAgePrebuildDays:         7,
 		},
-		EnableLocalApp:                  false,
+		EnableLocalApp:                  true,
 		AuthProviderConfigs:             ctx.Config.AuthProviders,
 		BuiltinAuthProvidersConfigured:  len(ctx.Config.AuthProviders) > 0,
 		DisableDynamicAuthProviderLogin: false,

diff --git a/installer/pkg/components/ws-daemon/clusterrole.go b/installer/pkg/components/ws-daemon/clusterrole.go
@@ -21,9 +21,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&rbacv1.ClusterRole{
 			TypeMeta: common.TypeMetaClusterRole,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-				Namespace: ctx.Namespace,
-				Labels:    labels,
+				Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+				Labels: labels,
 			},
 			Rules: []rbacv1.PolicyRule{{
 				APIGroups: []string{"policy"},

diff --git a/installer/pkg/components/ws-scheduler/clusterrole.go b/installer/pkg/components/ws-scheduler/clusterrole.go
@@ -21,9 +21,8 @@ func clusterrole(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&rbacv1.ClusterRole{
 			TypeMeta: common.TypeMetaClusterRole,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-				Namespace: ctx.Namespace,
-				Labels:    labels,
+				Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+				Labels: labels,
 			},
 			Rules: []rbacv1.PolicyRule{{
 				APIGroups: []string{""},

diff --git a/installer/pkg/components/ws-scheduler/clusterrolebinding.go b/installer/pkg/components/ws-scheduler/clusterrolebinding.go
@@ -21,9 +21,8 @@ func clusterrolebinding(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&rbacv1.ClusterRoleBinding{
 			TypeMeta: common.TypeMetaClusterRoleBinding,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
-				Namespace: ctx.Namespace,
-				Labels:    labels,
+				Name:   fmt.Sprintf("%s-ns-%s", ctx.Namespace, Component),
+				Labels: labels,
 			},
 			RoleRef: rbacv1.RoleRef{
 				Kind:     "ClusterRole",
@@ -41,9 +40,8 @@ func clusterrolebinding(ctx *common.RenderContext) ([]runtime.Object, error) {
 		&rbacv1.ClusterRoleBinding{
 			TypeMeta: common.TypeMetaClusterRoleBinding,
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      fmt.Sprintf("%s-ns-%s-kube-rbac-proxy", ctx.Namespace, Component),
-				Namespace: ctx.Namespace,
-				Labels:    labels,
+				Name:   fmt.Sprintf("%s-ns-%s-kube-rbac-proxy", ctx.Namespace, Component),
+				Labels: labels,
 			},
 			RoleRef: rbacv1.RoleRef{
 				Kind:     "ClusterRole",