From a2c59bcad2a41bcefae278ab75dd71f60e150969 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Tue, 29 Mar 2022 15:25:08 +0000
Subject: [PATCH 1/9] .werft/build: Decouple multiple kubeconfig paths

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/const.ts                    |  3 +++
 .../build/deploy-to-preview-environment.ts    |  7 ++-----
 .werft/jobs/build/prepare.ts                  | 11 ++++++-----
 .werft/vm/vm.ts                               | 19 +++++++++----------
 4 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/.werft/jobs/build/const.ts b/.werft/jobs/build/const.ts
index dedf0892753fb7..1a2e714409d1a8 100644
--- a/.werft/jobs/build/const.ts
+++ b/.werft/jobs/build/const.ts
@@ -1 +1,4 @@
 export const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
+export const CORE_DEV_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/core-dev";
+export const HARVESTER_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/harvester";
+export const PREVIEW_K3S_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/k3s";
diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index 7f32103d240f25..c9d25e4242ed55 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -6,7 +6,7 @@ import { InstallMonitoringSatelliteParams, installMonitoringSatellite } from '..
 import { wipeAndRecreateNamespace, setKubectlContextNamespace, deleteNonNamespaceObjects, findFreeHostPorts, createNamespace, helmInstallName, findLastHostPort, waitUntilAllPodsAreReady, waitForApiserver } from '../../util/kubectl';
 import { issueCertficate, installCertficate, IssueCertificateParams, InstallCertificateParams } from '../../util/certs';
 import { sleep, env } from '../../util/util';
-import { GCLOUD_SERVICE_ACCOUNT_PATH } from "./const";
+import { GCLOUD_SERVICE_ACCOUNT_PATH, PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
 import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
 import * as VM from '../../vm/vm'
@@ -122,10 +122,7 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         werft.done(vmSlices.START_KUBECTL_PORT_FORWARDS)
 
         werft.log(vmSlices.KUBECONFIG, 'Copying k3s kubeconfig')
-        VM.copyk3sKubeconfig({ name: destname, path: 'k3s.yml', timeoutMS: 1000 * 60 * 3, slice: vmSlices.KUBECONFIG })
-        // NOTE: This was a quick have to override the existing kubeconfig so all future kubectl commands use the k3s cluster.
-        //       We might want to keep both kubeconfigs around and be explicit about which one we're using.s
-        exec(`mv k3s.yml /home/gitpod/.kube/config`)
+        VM.copyk3sKubeconfig({ name: destname, timeoutMS: 1000 * 60 * 3, slice: vmSlices.KUBECONFIG })
         werft.done(vmSlices.KUBECONFIG)
 
         werft.log(vmSlices.WAIT_K3S, 'Wait for k3s')
diff --git a/.werft/jobs/build/prepare.ts b/.werft/jobs/build/prepare.ts
index f1dc9de29a7fc5..6df1d7f3b7274d 100644
--- a/.werft/jobs/build/prepare.ts
+++ b/.werft/jobs/build/prepare.ts
@@ -1,7 +1,7 @@
 import { exec } from '../../util/shell';
 import { Werft } from "../../util/werft";
 import * as VM from '../../vm/vm'
-import { GCLOUD_SERVICE_ACCOUNT_PATH } from "./const";
+import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, HARVESTER_KUBECONFIG_PATH } from "./const";
 import { JobConfig } from './job-config';
 
 const phaseName = "prepare";
@@ -17,7 +17,7 @@ export async function prepare(werft: Werft, config: JobConfig) {
         compareWerftAndGitpodImage()
         activateCoreDevServiceAccount()
         configureDocker()
-        configureCoreDevAccess()
+        configureStaticClustersAccess()
         werft.done(prepareSlices.CONFIGURE_CORE_DEV)
 
         decideHarvesterVMCreation(werft, config)
@@ -54,10 +54,11 @@ function configureDocker() {
     }
 }
 
-function configureCoreDevAccess() {
-    const rc = exec('gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev', { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
+function configureStaticClustersAccess() {
+    const rcCoreDev = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
+    const rcHarvester = exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
 
-    if (rc != 0) {
+    if (rcCoreDev != 0 || rcHarvester != 0) {
         throw new Error("Failed to get core-dev kubeconfig credentials.")
     }
 }
diff --git a/.werft/vm/vm.ts b/.werft/vm/vm.ts
index 2f49ea27c72ea0..31d20bbf21afd2 100644
--- a/.werft/vm/vm.ts
+++ b/.werft/vm/vm.ts
@@ -1,16 +1,15 @@
+import { HARVESTER_KUBECONFIG_PATH, PREVIEW_K3S_KUBECONFIG_PATH } from '../jobs/build/const';
 import { exec } from '../util/shell';
 import { getGlobalWerftInstance } from '../util/werft';
 
 import * as Manifests from './manifests'
 
-const KUBECONFIG_PATH = '/mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml'
-
 /**
  * Convenience function to kubectl apply a manifest from stdin.
  */
 function kubectlApplyManifest(manifest: string, options?: { validate?: boolean }) {
     exec(`
-        cat <<EOF | kubectl --kubeconfig ${KUBECONFIG_PATH} apply --validate=${!!options?.validate} -f -
+        cat <<EOF | kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} apply --validate=${!!options?.validate} -f -
 ${manifest}
 EOF
     `)
@@ -22,7 +21,7 @@ EOF
  */
  function kubectlDeleteManifest(manifest: string, options?: { validate?: boolean }) {
     exec(`
-        cat <<EOF | kubectl --kubeconfig ${KUBECONFIG_PATH} delete -f -
+        cat <<EOF | kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} delete -f -
 ${manifest}
 EOF
     `)
@@ -115,7 +114,7 @@ export function startVM(options: { name: string }) {
  */
 export function vmExists(options: { name: string }) {
     const namespace = `preview-${options.name}`
-    const status = exec(`kubectl --kubeconfig ${KUBECONFIG_PATH} -n ${namespace} get vmi ${options.name}`, { dontCheckRc: true, silent: true })
+    const status = exec(`kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} -n ${namespace} get vmi ${options.name}`, { dontCheckRc: true, silent: true })
     return status.code == 0
 }
 
@@ -128,7 +127,7 @@ export function waitForVMReadiness(options: { name: string, timeoutSeconds: numb
     const namespace = `preview-${options.name}`
 
     const startTime = Date.now()
-    const ready = exec(`kubectl --kubeconfig ${KUBECONFIG_PATH} -n ${namespace} wait --for=condition=ready --timeout=${options.timeoutSeconds}s pod -l kubevirt.io=virt-launcher -l harvesterhci.io/vmName=${options.name}`, { dontCheckRc: true, silent: true })
+    const ready = exec(`kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} -n ${namespace} wait --for=condition=ready --timeout=${options.timeoutSeconds}s pod -l kubevirt.io=virt-launcher -l harvesterhci.io/vmName=${options.name}`, { dontCheckRc: true, silent: true })
 
     if (ready.code == 0) {
         werft.log(options.slice, `VM is ready after ${(Date.now() - startTime) / 1000} seconds`)
@@ -143,15 +142,15 @@ export function waitForVMReadiness(options: { name: string, timeoutSeconds: numb
  * Copies the k3s kubeconfig out of the VM and places it at `path`
  * If it doesn't manage to do so before the timeout it will throw an Error
  */
-export function copyk3sKubeconfig(options: { name: string, path: string, timeoutMS: number, slice: string }) {
+export function copyk3sKubeconfig(options: { name: string, timeoutMS: number, slice: string }) {
     const werft = getGlobalWerftInstance()
     const startTime = Date.now()
     while (true) {
 
-        const status = exec(`ssh -i /workspace/.ssh/id_rsa_harvester_vm ubuntu@127.0.0.1 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 'sudo cat /etc/rancher/k3s/k3s.yaml' > ${options.path}`, { silent: true, dontCheckRc: true, slice: options.slice })
+        const status = exec(`ssh -i /workspace/.ssh/id_rsa_harvester_vm ubuntu@127.0.0.1 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 'sudo cat /etc/rancher/k3s/k3s.yaml' > ${PREVIEW_K3S_KUBECONFIG_PATH}`, { silent: true, dontCheckRc: true, slice: options.slice })
 
         if (status.code == 0) {
-            exec(`kubectl --kubeconfig ${options.path} config set clusters.default.server https://${options.name}.kube.gitpod-dev.com:6443`, { silent: true, slice: options.slice });
+            exec(`kubectl --kubeconfig ${PREVIEW_K3S_KUBECONFIG_PATH} config set clusters.default.server https://${options.name}.kube.gitpod-dev.com:6443`, { silent: true, slice: options.slice });
             return
         }
 
@@ -170,7 +169,7 @@ export function copyk3sKubeconfig(options: { name: string, path: string, timeout
  */
 export function startSSHProxy(options: { name: string, slice: string }) {
     const namespace = `preview-${options.name}`
-    exec(`sudo kubectl --kubeconfig=${KUBECONFIG_PATH} -n ${namespace} port-forward service/proxy 22:2200`, { async: true, silent: true, slice: options.slice, dontCheckRc: true })
+    exec(`sudo kubectl --kubeconfig=${HARVESTER_KUBECONFIG_PATH} -n ${namespace} port-forward service/proxy 22:2200`, { async: true, silent: true, slice: options.slice, dontCheckRc: true })
 }
 
 /**

From 41f9366bfcdc7191c36762b7795f924520a916a3 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Tue, 29 Mar 2022 17:13:41 +0000
Subject: [PATCH 2/9] .werft/deploy: Specify kubeconfig to every command

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .../build/deploy-to-preview-environment.ts    |  89 ++++++++-------
 .werft/jobs/build/installer/installer.ts      |  56 +++++-----
 .werft/observability/monitoring-satellite.ts  |  26 ++---
 .werft/util/certs.ts                          |   9 +-
 .werft/util/kubectl.ts                        | 105 +++++++++---------
 .werft/util/uninstall-gitpod.sh               |   7 +-
 .werft/vm/vm.ts                               |  14 +--
 7 files changed, 157 insertions(+), 149 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index c9d25e4242ed55..836c8272c37ae3 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -6,7 +6,7 @@ import { InstallMonitoringSatelliteParams, installMonitoringSatellite } from '..
 import { wipeAndRecreateNamespace, setKubectlContextNamespace, deleteNonNamespaceObjects, findFreeHostPorts, createNamespace, helmInstallName, findLastHostPort, waitUntilAllPodsAreReady, waitForApiserver } from '../../util/kubectl';
 import { issueCertficate, installCertficate, IssueCertificateParams, InstallCertificateParams } from '../../util/certs';
 import { sleep, env } from '../../util/util';
-import { GCLOUD_SERVICE_ACCOUNT_PATH, PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
+import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
 import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
 import * as VM from '../../vm/vm'
@@ -99,18 +99,18 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         withVM,
     };
 
-    exec(`kubectl --namespace keys get secret host-key -o yaml > /workspace/host-key.yaml`)
+    exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} --namespace keys get secret host-key -o yaml > /workspace/host-key.yaml`)
 
     // Writing auth-provider configuration to disk prior to deploying anything.
     // We do this because we have different auth-providers depending if we're using core-dev or Harvester VMs.
-    exec(`kubectl get secret ${withVM ? 'preview-envs-authproviders-harvester' : 'preview-envs-authproviders'} --namespace=keys -o jsonpath="{.data.authProviders}" > auth-provider-secret.yml`, { silent: true })
+    exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret ${withVM ? 'preview-envs-authproviders-harvester' : 'preview-envs-authproviders'} --namespace=keys -o jsonpath="{.data.authProviders}" > auth-provider-secret.yml`, { silent: true })
 
     if (withVM) {
         werft.phase(phases.VM, "Ensuring VM is ready for deployment");
 
         werft.log(vmSlices.COPY_CERT_MANAGER_RESOURCES, 'Copy over CertManager resources from core-dev')
-        exec(`kubectl get secret clouddns-dns01-solver-svc-acct -n certmanager -o yaml | sed 's/namespace: certmanager/namespace: cert-manager/g' > clouddns-dns01-solver-svc-acct.yaml`, { slice: vmSlices.COPY_CERT_MANAGER_RESOURCES })
-        exec(`kubectl get clusterissuer letsencrypt-issuer-gitpod-core-dev -o yaml | sed 's/letsencrypt-issuer-gitpod-core-dev/letsencrypt-issuer/g' > letsencrypt-issuer.yaml`, { slice: vmSlices.COPY_CERT_MANAGER_RESOURCES })
+        exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret clouddns-dns01-solver-svc-acct -n certmanager -o yaml | sed 's/namespace: certmanager/namespace: cert-manager/g' > clouddns-dns01-solver-svc-acct.yaml`, { slice: vmSlices.COPY_CERT_MANAGER_RESOURCES })
+        exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get clusterissuer letsencrypt-issuer-gitpod-core-dev -o yaml | sed 's/letsencrypt-issuer-gitpod-core-dev/letsencrypt-issuer/g' > letsencrypt-issuer.yaml`, { slice: vmSlices.COPY_CERT_MANAGER_RESOURCES })
         werft.done(vmSlices.COPY_CERT_MANAGER_RESOURCES)
 
         werft.log(vmSlices.VM_READINESS, 'Wait for VM readiness')
@@ -126,24 +126,24 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         werft.done(vmSlices.KUBECONFIG)
 
         werft.log(vmSlices.WAIT_K3S, 'Wait for k3s')
-        await waitForApiserver({ slice: vmSlices.WAIT_K3S })
-        await waitUntilAllPodsAreReady("kube-system", { slice: vmSlices.WAIT_K3S })
+        await waitForApiserver(PREVIEW_K3S_KUBECONFIG_PATH, { slice: vmSlices.WAIT_K3S })
+        await waitUntilAllPodsAreReady("kube-system", PREVIEW_K3S_KUBECONFIG_PATH, { slice: vmSlices.WAIT_K3S })
         werft.done(vmSlices.WAIT_K3S)
 
         werft.log(vmSlices.WAIT_CERTMANAGER, 'Wait for Cert-Manager')
-        await waitUntilAllPodsAreReady("cert-manager", { slice: vmSlices.WAIT_CERTMANAGER })
+        await waitUntilAllPodsAreReady("cert-manager", PREVIEW_K3S_KUBECONFIG_PATH, { slice: vmSlices.WAIT_CERTMANAGER })
         werft.done(vmSlices.WAIT_CERTMANAGER)
 
-        exec(`kubectl apply -f clouddns-dns01-solver-svc-acct.yaml -f letsencrypt-issuer.yaml`, { slice: vmSlices.INSTALL_LETS_ENCRYPT_ISSUER, dontCheckRc: true })
+        exec(`kubectl --kubeconfig ${PREVIEW_K3S_KUBECONFIG_PATH} apply -f clouddns-dns01-solver-svc-acct.yaml -f letsencrypt-issuer.yaml`, { slice: vmSlices.INSTALL_LETS_ENCRYPT_ISSUER, dontCheckRc: true })
         werft.done(vmSlices.INSTALL_LETS_ENCRYPT_ISSUER)
 
-        VM.installFluentBit({ namespace: 'default', slice: vmSlices.EXTERNAL_LOGGING })
+        VM.installFluentBit({ namespace: 'default', kubeconfig: PREVIEW_K3S_KUBECONFIG_PATH, slice: vmSlices.EXTERNAL_LOGGING })
         werft.done(vmSlices.EXTERNAL_LOGGING)
 
 
         issueMetaCerts(werft, PROXY_SECRET_NAME, "default", domain, withVM)
         werft.done('certificate')
-        installMonitoring(deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch);
+        installMonitoring(PREVIEW_K3S_KUBECONFIG_PATH, deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch);
         werft.done('observability')
     }
 
@@ -190,12 +190,13 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     // to test this function, change files in your workspace, sideload (-s) changed files into werft or set annotations (-a) like so:
     // werft run github -f -j ./.werft/build.yaml -s ./.werft/build.ts -s ./.werft/jobs/build/installer/post-process.sh -a with-clean-slate-deployment=true
     const { version, destname, namespace, domain, monitoringDomain, url, withObservability, withVM } = deploymentConfig;
+    const deploymentKubeconfig = withVM ? PREVIEW_K3S_KUBECONFIG_PATH : CORE_DEV_KUBECONFIG_PATH;
 
     // find free ports
     werft.log(installerSlices.FIND_FREE_HOST_PORTS, "Find last ports");
-    let wsdaemonPortMeta = findLastHostPort(namespace, 'ws-daemon', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
-    let registryNodePortMeta = findLastHostPort(namespace, 'registry-facade', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
-    let nodeExporterPort = findLastHostPort(namespace, 'node-exporter', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
+    let wsdaemonPortMeta = findLastHostPort(namespace, 'ws-daemon', deploymentKubeconfig, metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
+    let registryNodePortMeta = findLastHostPort(namespace, 'registry-facade', deploymentKubeconfig, metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
+    let nodeExporterPort = findLastHostPort(namespace, 'node-exporter', deploymentKubeconfig, metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
 
     if (isNaN(wsdaemonPortMeta) || isNaN(wsdaemonPortMeta) || (isNaN(nodeExporterPort) && !withVM && withObservability)) {
         werft.log(installerSlices.FIND_FREE_HOST_PORTS, "Can't reuse, check for some free ports.");
@@ -203,7 +204,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
             { start: 10000, end: 11000 },
             { start: 30000, end: 31000 },
             { start: 31001, end: 32000 },
-        ], metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }));
+        ], deploymentKubeconfig, metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }));
     }
     werft.log(installerSlices.FIND_FREE_HOST_PORTS,
         `wsdaemonPortMeta: ${wsdaemonPortMeta}, registryNodePortMeta: ${registryNodePortMeta}.`);
@@ -218,7 +219,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
 
         } else {
             werft.log(installerSlices.CLEAN_ENV_STATE, "Clean the preview environment slate...");
-            createNamespace(namespace, metaEnv({ slice: installerSlices.CLEAN_ENV_STATE }));
+            createNamespace(namespace, deploymentKubeconfig, metaEnv({ slice: installerSlices.CLEAN_ENV_STATE }));
         }
         werft.done(installerSlices.CLEAN_ENV_STATE);
     } catch (err) {
@@ -249,13 +250,13 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     }
 
     // add the image pull secret to the namespcae if it doesn't exist
-    const hasPullSecret = (exec(`kubectl get secret ${IMAGE_PULL_SECRET_NAME} -n ${namespace}`, { slice: installerSlices.IMAGE_PULL_SECRET, dontCheckRc: true, silent: true })).code === 0;
+    const hasPullSecret = (exec(`kubectl --kubeconfig ${deploymentKubeconfig} get secret ${IMAGE_PULL_SECRET_NAME} -n ${namespace}`, { slice: installerSlices.IMAGE_PULL_SECRET, dontCheckRc: true, silent: true })).code === 0;
     if (!hasPullSecret) {
         try {
             werft.log(installerSlices.IMAGE_PULL_SECRET, "Adding the image pull secret to the namespace");
             const dockerConfig = { auths: { "eu.gcr.io": { auth: deploymentConfig.imagePullAuth }, "europe-docker.pkg.dev": { auth: deploymentConfig.imagePullAuth } } };
             fs.writeFileSync(`./${IMAGE_PULL_SECRET_NAME}`, JSON.stringify(dockerConfig));
-            exec(`kubectl create secret docker-registry ${IMAGE_PULL_SECRET_NAME} -n ${namespace} --from-file=.dockerconfigjson=./${IMAGE_PULL_SECRET_NAME}`, { slice: installerSlices.IMAGE_PULL_SECRET });
+            exec(`kubectl --kubeconfig ${deploymentKubeconfig} create secret docker-registry ${IMAGE_PULL_SECRET_NAME} -n ${namespace} --from-file=.dockerconfigjson=./${IMAGE_PULL_SECRET_NAME}`, { slice: installerSlices.IMAGE_PULL_SECRET });
         }
         catch (err) {
             if (!jobConfig.mainBuild) {
@@ -274,7 +275,8 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
 
     const installer = new Installer({
         werft: werft,
-        configPath: "config.yaml",
+        installerConfigPath: "config.yaml",
+        kubeconfigPath: deploymentKubeconfig,
         version: version,
         proxySecretName: PROXY_SECRET_NAME,
         domain: deploymentConfig.domain,
@@ -302,10 +304,10 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     }
 
     werft.log(installerSlices.DEPLOYMENT_WAITING, "Waiting until all pods are ready.");
-    await waitUntilAllPodsAreReady(deploymentConfig.namespace, { slice: installerSlices.DEPLOYMENT_WAITING })
+    await waitUntilAllPodsAreReady(deploymentConfig.namespace, installer.options.kubeconfigPath, { slice: installerSlices.DEPLOYMENT_WAITING })
     werft.done(installerSlices.DEPLOYMENT_WAITING);
 
-    await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, !withVM)
+    await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, !withVM, installer.options.kubeconfigPath)
 
     // TODO: Fix sweeper, it does not appear to be doing clean-up
     werft.log('sweeper', 'installing Sweeper');
@@ -326,23 +328,23 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     // TODO: Implement sweeper logic for VMs in Harvester
     if (!withVM) {
         // copy GH token into namespace
-        exec(`kubectl --namespace werft get secret github-sweeper-read-branches -o yaml \
+        exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} --namespace werft get secret github-sweeper-read-branches -o yaml \
             | yq w - metadata.namespace ${namespace} \
             | yq d - metadata.uid \
             | yq d - metadata.resourceVersion \
             | yq d - metadata.creationTimestamp \
-            | kubectl apply -f -`);
-        exec(`/usr/local/bin/helm3 upgrade --install --set image.version=${sweeperVersion} --set command="werft run github -a namespace=${namespace} --remote-job-path .werft/wipe-devstaging.yaml github.com/gitpod-io/gitpod:main" ${allArgsStr} sweeper ./dev/charts/sweeper`);
+            | kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} apply -f -`);
+        exec(`/usr/local/bin/helm3 --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} upgrade --install --set image.version=${sweeperVersion} --set command="werft run github -a namespace=${namespace} --remote-job-path .werft/wipe-devstaging.yaml github.com/gitpod-io/gitpod:main" ${allArgsStr} sweeper ./dev/charts/sweeper`);
     }
 
     werft.done(phases.DEPLOY);
 
     async function cleanStateEnv(shellOpts: ExecOptions) {
-        await wipeAndRecreateNamespace(helmInstallName, namespace, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
+        await wipeAndRecreateNamespace(helmInstallName, namespace, installer.options.kubeconfigPath, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
         // cleanup non-namespace objects
         werft.log(installerSlices.CLEAN_ENV_STATE, "removing old unnamespaced objects - this might take a while");
         try {
-            await deleteNonNamespaceObjects(namespace, destname, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
+            await deleteNonNamespaceObjects(namespace, destname, installer.options.kubeconfigPath, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
             werft.done(installerSlices.CLEAN_ENV_STATE);
         } catch (err) {
             werft.fail(installerSlices.CLEAN_ENV_STATE, err);
@@ -361,7 +363,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         { start: 10000, end: 11000 },
         { start: 30000, end: 31000 },
         { start: 31001, end: 32000 },
-    ], metaEnv({ slice: "find free ports", silent: true }));
+    ], CORE_DEV_KUBECONFIG_PATH, metaEnv({ slice: "find free ports", silent: true }));
     werft.log("find free ports",
         `wsdaemonPortMeta: ${wsdaemonPortMeta}, registryNodePortMeta: ${registryNodePortMeta}, and nodeExporterPort ${nodeExporterPort}.`);
     werft.done("find free ports");
@@ -378,7 +380,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
             // re-create namespace
             await cleanStateEnv(metaEnv());
         } else {
-            createNamespace(namespace, metaEnv({ slice: 'prep' }));
+            createNamespace(namespace, CORE_DEV_KUBECONFIG_PATH, metaEnv({ slice: 'prep' }));
         }
         // Now we want to execute further kubectl operations only in the created namespace
         setKubectlContextNamespace(namespace, metaEnv({ slice: 'prep' }));
@@ -388,7 +390,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         await issueMetaCerts(werft, namespace, "certs", domain, false);
         await installMetaCertificates(werft, namespace);
         werft.done('certificate');
-        await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, false)
+        await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, false, CORE_DEV_KUBECONFIG_PATH)
         werft.done('prep');
     } catch (err) {
         if (!jobConfig.mainBuild) {
@@ -400,7 +402,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
     // core-dev specific section start
     werft.log("secret", "copy secret into namespace")
     try {
-        const auth = exec(`printf "%s" "_json_key:$(kubectl get secret ${IMAGE_PULL_SECRET_NAME} --namespace=keys -o yaml \
+        const auth = exec(`printf "%s" "_json_key:$(kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret ${IMAGE_PULL_SECRET_NAME} --namespace=keys -o yaml \
                         | yq r - data['.dockerconfigjson'] \
                         | base64 -d)" | base64 -w 0`, { silent: true }).stdout.trim();
         fs.writeFileSync("chart/gcp-sa-registry-auth",
@@ -424,7 +426,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
 
     werft.log("authProviders", "copy authProviders")
     try {
-        exec(`kubectl get secret preview-envs-authproviders --namespace=keys -o yaml \
+        exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret preview-envs-authproviders --namespace=keys -o yaml \
                 | yq r - data.authProviders \
                 | base64 -d -w 0 \
                 > authProviders`, { slice: "authProviders" });
@@ -446,7 +448,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
     werft.log(`observability`, "Installing monitoring-satellite...")
     if (deploymentConfig.withObservability) {
         try {
-            await installMonitoring(namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, false, jobConfig.observability.branch);
+            await installMonitoring(CORE_DEV_KUBECONFIG_PATH ,namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, false, jobConfig.observability.branch);
         } catch (err) {
             if (!jobConfig.mainBuild) {
                 werft.fail('observability', err);
@@ -487,7 +489,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         const nodeAffinityValues = getNodeAffinities();
 
         if (storage === "gcp") {
-            exec("kubectl get secret gcp-sa-gitpod-dev-deployer -n werft -o yaml | yq d - metadata | yq w - metadata.name remote-storage-gcloud | kubectl apply -f -");
+            exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret gcp-sa-gitpod-dev-deployer -n werft -o yaml | yq d - metadata | yq w - metadata.name remote-storage-gcloud | kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} apply -f -`);
             flags += ` -f ../.werft/jobs/build/helm/values.dev.gcp-storage.yaml`;
         }
 
@@ -499,7 +501,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         const nodepoolIndex = getNodePoolIndex(namespace);
 
         exec(`helm dependencies up`);
-        exec(`/usr/local/bin/helm3 upgrade --install --timeout 10m -f ../.werft/jobs/build/helm/${nodeAffinityValues[nodepoolIndex]} -f ../.werft/jobs/build/helm/values.dev.yaml ${flags} ${helmInstallName} .`);
+        exec(`/usr/local/bin/helm3 --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} upgrade --install --timeout 10m -f ../.werft/jobs/build/helm/${nodeAffinityValues[nodepoolIndex]} -f ../.werft/jobs/build/helm/values.dev.yaml ${flags} ${helmInstallName} .`);
 
         werft.log('helm', 'installing Sweeper');
         const sweeperVersion = deploymentConfig.sweeperImage.split(":")[1];
@@ -517,13 +519,13 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         const allArgsStr = `--set args="{${argsStr}}" --set githubToken.secret=github-sweeper-read-branches --set githubToken.key=token`;
 
         // copy GH token into namespace
-        exec(`kubectl --namespace werft get secret github-sweeper-read-branches -o yaml \
+        exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} --namespace werft get secret github-sweeper-read-branches -o yaml \
             | yq w - metadata.namespace ${namespace} \
             | yq d - metadata.uid \
             | yq d - metadata.resourceVersion \
             | yq d - metadata.creationTimestamp \
-            | kubectl apply -f -`);
-        exec(`/usr/local/bin/helm3 upgrade --install --set image.version=${sweeperVersion} --set command="werft run github -a namespace=${namespace} --remote-job-path .werft/wipe-devstaging.yaml github.com/gitpod-io/gitpod:main" ${allArgsStr} sweeper ../dev/charts/sweeper`);
+            | kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} apply -f -`);
+        exec(`/usr/local/bin/helm3 --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} upgrade --install --set image.version=${sweeperVersion} --set command="werft run github -a namespace=${namespace} --remote-job-path .werft/wipe-devstaging.yaml github.com/gitpod-io/gitpod:main" ${allArgsStr} sweeper ../dev/charts/sweeper`);
     }
 
     function addDeploymentFlags() {
@@ -577,11 +579,11 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
     }
 
     async function cleanStateEnv(shellOpts: ExecOptions) {
-        await wipeAndRecreateNamespace(helmInstallName, namespace, { ...shellOpts, slice: 'prep' });
+        await wipeAndRecreateNamespace(helmInstallName, namespace, CORE_DEV_KUBECONFIG_PATH, { ...shellOpts, slice: 'prep' });
         // cleanup non-namespace objects
         werft.log("predeploy cleanup", "removing old unnamespaced objects - this might take a while");
         try {
-            await deleteNonNamespaceObjects(namespace, destname, { ...shellOpts, slice: 'predeploy cleanup' });
+            await deleteNonNamespaceObjects(namespace, destname, CORE_DEV_KUBECONFIG_PATH, { ...shellOpts, slice: 'predeploy cleanup' });
             werft.done('predeploy cleanup');
         } catch (err) {
             if (!jobConfig.mainBuild) {
@@ -631,13 +633,13 @@ interface DeploymentConfig {
     withVM: boolean;
 }
 
-async function addDNSRecord(werft: Werft, namespace: string, domain: string, isLoadbalancer: boolean) {
+async function addDNSRecord(werft: Werft, namespace: string, domain: string, isLoadbalancer: boolean, kubeconfigPath: string) {
     let wsProxyLBIP = null
     if (isLoadbalancer === true) {
         werft.log(installerSlices.DNS_ADD_RECORD, "Getting ws-proxy loadbalancer IP");
         for (let i = 0; i < 60; i++) {
             try {
-                let lb = exec(`kubectl -n ${namespace} get service ws-proxy -o=jsonpath='{.status.loadBalancer.ingress[0].ip}'`, { silent: true })
+                let lb = exec(`kubectl --kubeconfig ${kubeconfigPath} -n ${namespace} get service ws-proxy -o=jsonpath='{.status.loadBalancer.ingress[0].ip}'`, { silent: true })
                 if (lb.length > 4) {
                     wsProxyLBIP = lb
                     break
@@ -696,10 +698,11 @@ async function installMetaCertificates(werft: Werft, namespace: string) {
     await installCertficate(werft, metaInstallCertParams, metaEnv());
 }
 
-async function installMonitoring(namespace: string, nodeExporterPort: number, domain: string, stackdriverServiceAccount: any, withVM: boolean, observabilityBranch: string) {
+async function installMonitoring(kubeconfig: string, namespace: string, nodeExporterPort: number, domain: string, stackdriverServiceAccount: any, withVM: boolean, observabilityBranch: string) {
     const installMonitoringSatelliteParams = new InstallMonitoringSatelliteParams();
+    installMonitoringSatelliteParams.kubeconfigPath = kubeconfig
     installMonitoringSatelliteParams.branch = observabilityBranch;
-    installMonitoringSatelliteParams.pathToKubeConfig = ""
+    installMonitoringSatelliteParams.kubeconfigPath = ""
     installMonitoringSatelliteParams.satelliteNamespace = namespace
     installMonitoringSatelliteParams.clusterName = namespace
     installMonitoringSatelliteParams.nodeExporterPort = nodeExporterPort
diff --git a/.werft/jobs/build/installer/installer.ts b/.werft/jobs/build/installer/installer.ts
index 45b003717cbeb1..257466b2e0defa 100644
--- a/.werft/jobs/build/installer/installer.ts
+++ b/.werft/jobs/build/installer/installer.ts
@@ -20,7 +20,8 @@ export type GitpodDaemonsetPorts = {
 
 export type InstallerOptions = {
     werft: Werft
-    configPath: string
+    installerConfigPath: string
+    kubeconfigPath: string
     version: string
     proxySecretName: string
     domain: string
@@ -45,7 +46,7 @@ export class Installer {
         this.options.werft.log(slice, "Downloading installer and initializing config file");
         exec(`docker run --entrypoint sh --rm eu.gcr.io/gitpod-core-dev/build/installer:${this.options.version} -c "cat /app/installer" > /tmp/installer`, { slice: slice });
         exec(`chmod +x /tmp/installer`, { slice: slice });
-        exec(`/tmp/installer init > ${this.options.configPath}`, { slice: slice });
+        exec(`/tmp/installer init > ${this.options.installerConfigPath}`, { slice: slice });
         this.options.werft.done(slice);
     }
 
@@ -75,30 +76,30 @@ export class Installer {
         exec(`yq r ./.werft/jobs/build/helm/values.dev.yaml components.server.blockNewUsers | yq prefix - 'blockNewUsers' > ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
         exec(`yq r ./.werft/jobs/build/helm/values.variant.cpuLimits.yaml workspaceSizing.dynamic.cpu.buckets | yq prefix - 'workspace.resources.dynamicLimits.cpu' > ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });
 
-        exec(`yq m -i --overwrite ${this.options.configPath} ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
-        exec(`yq m -i ${this.options.configPath} ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });
+        exec(`yq m -i --overwrite ${this.options.installerConfigPath} ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
+        exec(`yq m -i ${this.options.installerConfigPath} ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });
     }
 
     private configureContainerRegistry(slice: string): void {
-        exec(`yq w -i ${this.options.configPath} certificate.name ${this.options.proxySecretName}`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} containerRegistry.inCluster false`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} containerRegistry.external.url ${CONTAINER_REGISTRY_URL}`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} containerRegistry.external.certificate.kind secret`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} containerRegistry.external.certificate.name ${this.options.imagePullSecretName}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} certificate.name ${this.options.proxySecretName}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.inCluster false`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.url ${CONTAINER_REGISTRY_URL}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.certificate.kind secret`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.certificate.name ${this.options.imagePullSecretName}`, { slice: slice });
     }
 
     private configureDomain(slice: string) {
-        exec(`yq w -i ${this.options.configPath} domain ${this.options.domain}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} domain ${this.options.domain}`, { slice: slice });
     }
 
     private configureWorkspaces(slice: string) {
-        exec(`yq w -i ${this.options.configPath} workspace.runtime.containerdRuntimeDir ${CONTAINERD_RUNTIME_DIR}`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} workspace.resources.requests.cpu "100m"`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} workspace.runtime.containerdRuntimeDir ${CONTAINERD_RUNTIME_DIR}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} workspace.resources.requests.cpu "100m"`, { slice: slice });
     }
 
     private configureObservability(slice: string) {
         const tracingEndpoint = exec(`yq r ./.werft/jobs/build/helm/values.tracing.yaml tracing.endpoint`, { slice: slice }).stdout.trim();
-        exec(`yq w -i ${this.options.configPath} observability.tracing.endpoint ${tracingEndpoint}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} observability.tracing.endpoint ${tracingEndpoint}`, { slice: slice });
     }
 
     // auth-provider-secret.yml is a file generated by this job by reading a secret from core-dev cluster
@@ -114,14 +115,15 @@ export class Installer {
             providerId=$(echo $row | base64 -d | jq -r '.value.id | ascii_downcase')
             data=$(echo $row | base64 -d | yq r - value --prettyPrint)
 
-            yq w -i ${this.options.configPath} authProviders[$key].kind "secret"
-            yq w -i ${this.options.configPath} authProviders[$key].name "$providerId"
+            yq w -i ${this.options.installerConfigPath} authProviders[$key].kind "secret"
+            yq w -i ${this.options.installerConfigPath} authProviders[$key].name "$providerId"
 
             kubectl create secret generic "$providerId" \
                 --namespace "${this.options.deploymentNamespace}" \
+                --kubeconfig "${this.options.kubeconfigPath}" \
                 --from-literal=provider="$data" \
                 --dry-run=client -o yaml | \
-                kubectl replace --force -f -
+                kubectl --kubeconfig "${this.options.kubeconfigPath}" replace --force -f -
         done`, { slice: slice })
     }
 
@@ -131,30 +133,30 @@ export class Installer {
                 | yq d - metadata.uid \
                 | yq d - metadata.resourceVersion \
                 | yq d - metadata.creationTimestamp \
-                | kubectl apply -f -`, { slice: slice })
-        exec(`yq w -i ${this.options.configPath} sshGatewayHostKey.kind "secret"`)
-        exec(`yq w -i ${this.options.configPath} sshGatewayHostKey.name "host-key"`)
+                | kubectl --kubeconfig ${this.options.kubeconfigPath} apply -f -`, { slice: slice })
+        exec(`yq w -i ${this.options.installerConfigPath} sshGatewayHostKey.kind "secret"`)
+        exec(`yq w -i ${this.options.installerConfigPath} sshGatewayHostKey.name "host-key"`)
     }
 
     private includeAnalytics(slice: string): void {
-        exec(`yq w -i ${this.options.configPath} analytics.writer segment`, { slice: slice });
-        exec(`yq w -i ${this.options.configPath} analytics.segmentKey ${this.options.analytics.token}`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} analytics.writer segment`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} analytics.segmentKey ${this.options.analytics.token}`, { slice: slice });
     }
 
     private dontIncludeAnalytics(slice: string): void {
-        exec(`yq w -i ${this.options.configPath} analytics.writer ""`, { slice: slice });
+        exec(`yq w -i ${this.options.installerConfigPath} analytics.writer ""`, { slice: slice });
     }
 
     validateConfiguration(slice: string): void {
         this.options.werft.log(slice, "Validating configuration");
-        exec(`/tmp/installer validate config -c ${this.options.configPath}`, { slice: slice });
-        exec(`/tmp/installer validate cluster -c ${this.options.configPath} || true`, { slice: slice });
+        exec(`/tmp/installer validate config -c ${this.options.installerConfigPath}`, { slice: slice });
+        exec(`/tmp/installer validate cluster --kubeconfig ${this.options.kubeconfigPath} -c ${this.options.installerConfigPath} || true`, { slice: slice });
         this.options.werft.done(slice)
     }
 
     render(slice: string): void {
         this.options.werft.log(slice, "Rendering YAML manifests");
-        exec(`/tmp/installer render --namespace ${this.options.deploymentNamespace} --config ${this.options.configPath} > k8s.yaml`, { slice: slice });
+        exec(`/tmp/installer render --namespace ${this.options.deploymentNamespace} --config ${this.options.installerConfigPath} > k8s.yaml`, { slice: slice });
         this.options.werft.done(slice)
     }
 
@@ -199,9 +201,9 @@ export class Installer {
 
     install(slice: string): void {
         this.options.werft.log(slice, "Installing Gitpod");
-        exec(`kubectl delete -n ${this.options.deploymentNamespace} job migrations || true`, { silent: true });
+        exec(`kubectl --kubeconfig ${this.options.kubeconfigPath} delete -n ${this.options.deploymentNamespace} job migrations || true`, { silent: true });
         // errors could result in outputing a secret to the werft log when kubernetes patches existing objects...
-        exec(`kubectl apply -f k8s.yaml`, { silent: true });
+        exec(`kubectl --kubeconfig ${this.options.kubeconfigPath} apply -f k8s.yaml`, { silent: true });
 
         exec(`werft log result -d "dev installation" -c github-check-preview-env url https://${this.options.domain}/workspaces`);
         this.options.werft.done(slice)
diff --git a/.werft/observability/monitoring-satellite.ts b/.werft/observability/monitoring-satellite.ts
index 3d73d15ff3bc75..d764118c5de255 100644
--- a/.werft/observability/monitoring-satellite.ts
+++ b/.werft/observability/monitoring-satellite.ts
@@ -7,7 +7,7 @@ import * as fs from 'fs';
  * Monitoring satellite deployment bits
  */
  export class InstallMonitoringSatelliteParams {
-    pathToKubeConfig: string
+    kubeconfigPath: string
     satelliteNamespace: string
     clusterName: string
     nodeExporterPort: number
@@ -76,39 +76,39 @@ export async function installMonitoringSatellite(params: InstallMonitoringSatell
 
     // The correct kubectl context should already be configured prior to this step
     // Only checks node-exporter readiness for harvester
-    ensureCorrectInstallationOrder(params.satelliteNamespace, params.withVM)
+    ensureCorrectInstallationOrder(params.kubeconfigPath, params.satelliteNamespace, params.withVM)
 }
 
-async function ensureCorrectInstallationOrder(namespace: string, checkNodeExporterStatus: boolean){
+async function ensureCorrectInstallationOrder(kubeconfig: string, namespace: string, checkNodeExporterStatus: boolean){
     const werft = getGlobalWerftInstance()
 
     werft.log(sliceName, 'installing monitoring-satellite')
-    exec('cd observability && hack/deploy-satellite.sh', {slice: sliceName})
+    exec(`cd observability && KUBECONFIG=${kubeconfig} hack/deploy-satellite.sh`, {slice: sliceName})
 
-    deployGitpodServiceMonitors()
-    checkReadiness(namespace, checkNodeExporterStatus)
+    deployGitpodServiceMonitors(kubeconfig)
+    checkReadiness(kubeconfig, namespace, checkNodeExporterStatus)
 }
 
-async function checkReadiness(namespace: string, checkNodeExporterStatus: boolean) {
+async function checkReadiness(kubeconfig: string, namespace: string, checkNodeExporterStatus: boolean) {
     // For some reason prometheus' statefulset always take quite some time to get created
     // Therefore we wait a couple of seconds
     exec(`sleep 30 && kubectl rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
-    exec(`kubectl rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
-    exec(`kubectl rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
-    exec(`kubectl rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})
+    exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
+    exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
+    exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})
 
     // core-dev is just too unstable for node-exporter
     // we don't guarantee that it will run at all
     if(checkNodeExporterStatus) {
-        exec(`kubectl rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
+        exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
     }
 }
 
-async function deployGitpodServiceMonitors() {
+async function deployGitpodServiceMonitors(kubeconfig: string) {
     const werft = getGlobalWerftInstance()
 
     werft.log(sliceName, 'installing gitpod ServiceMonitor resources')
-    exec('kubectl apply -f observability/monitoring-satellite/manifests/gitpod/', {silent: true})
+    exec(`kubectl --kubeconfig ${kubeconfig} apply -f observability/monitoring-satellite/manifests/gitpod/`, {silent: true})
 }
 
 function postProcessManifests() {
diff --git a/.werft/util/certs.ts b/.werft/util/certs.ts
index e2e47c09b24edf..35c4b78e409a06 100644
--- a/.werft/util/certs.ts
+++ b/.werft/util/certs.ts
@@ -1,7 +1,7 @@
 import { exec, ExecOptions } from './shell';
 import { sleep } from './util';
-import { readFileSync, writeFileSync } from 'fs';
 import * as path from 'path';
+import { CORE_DEV_KUBECONFIG_PATH } from '../jobs/build/const';
 
 
 export class IssueCertificateParams {
@@ -38,6 +38,9 @@ export async function issueCertficate(werft, params: IssueCertificateParams, she
     })) {
         throw new Error(`there is no subdomain + '${params.domain}' shorter or equal to 63 characters, max. allowed length for CN. No HTTPS certs for you! Consider using a short branch name...`);
     }
+
+    // Certificates are always issued in the core-dev cluster.
+    // They might be copied to other clusters in future steps.
     var cmd = `set -x \
     && cd ${path.join(params.pathToTemplate)} \
     && cp cert-manager_certificate.tpl cert.yaml \
@@ -45,7 +48,7 @@ export async function issueCertficate(werft, params: IssueCertificateParams, she
     && yq w -i cert.yaml spec.secretName '${params.namespace}' \
     && yq w -i cert.yaml metadata.namespace '${params.certNamespace}' \
     ${subdomains.map(s => `&& yq w -i cert.yaml spec.dnsNames[+] '${s+params.domain}'`).join('  ')} \
-    && kubectl apply -f cert.yaml`;
+    && kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} apply -f cert.yaml`;
 
     werft.log("certificate", "Kubectl command for cert creation: " + cmd)
     exec(cmd, { ...shellOpts, slice: 'certificate' });
@@ -54,7 +57,7 @@ export async function issueCertficate(werft, params: IssueCertificateParams, she
     let notReadyYet = true;
     for (let i = 0; i < 90 && notReadyYet; i++) {
         werft.log('certificate', `polling state of ${params.certNamespace}/${params.namespace}...`)
-        const result = exec(`kubectl -n ${params.certNamespace} get certificate ${params.namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { ...shellOpts, silent: true, dontCheckRc: true, async: false });
+        const result = exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} -n ${params.certNamespace} get certificate ${params.namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { ...shellOpts, silent: true, dontCheckRc: true, async: false });
         if (result != undefined && result.code === 0 && result.stdout === "True") {
             notReadyYet = false;
             break;
diff --git a/.werft/util/kubectl.ts b/.werft/util/kubectl.ts
index 4598e060d986ee..2f67b76cd82bd3 100644
--- a/.werft/util/kubectl.ts
+++ b/.werft/util/kubectl.ts
@@ -1,7 +1,6 @@
-import { ShellString } from 'shelljs';
 import { exec, ExecOptions } from './shell';
 import { sleep } from './util';
-import { getGlobalWerftInstance, Werft } from './werft';
+import { getGlobalWerftInstance } from './werft';
 
 
 export const IS_PREVIEW_APP_LABEL: string = "isPreviewApp";
@@ -15,50 +14,50 @@ export function setKubectlContextNamespace(namespace: string, shellOpts: ExecOpt
     ].forEach(cmd => exec(cmd, shellOpts));
 }
 
-export async function wipePreviewEnvironmentAndNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+export async function wipePreviewEnvironmentAndNamespace(helmInstallName: string, namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
     const werft = getGlobalWerftInstance();
 
     // wipe preview envs built with installer
-    await wipePreviewEnvironmentInstaller(namespace, shellOpts);
+    await wipePreviewEnvironmentInstaller(namespace, kubeconfig, shellOpts);
 
     // wipe preview envs previously built with helm
-    await wipePreviewEnvironmentHelm(helmInstallName, namespace, shellOpts)
+    await wipePreviewEnvironmentHelm(helmInstallName, namespace, kubeconfig, shellOpts)
 
-    deleteAllWorkspaces(namespace, shellOpts);
+    deleteAllWorkspaces(namespace, kubeconfig, shellOpts);
 
-    await deleteAllUnnamespacedObjects(namespace, shellOpts);
+    await deleteAllUnnamespacedObjects(namespace, kubeconfig, shellOpts);
 
-    deleteNamespace(true, namespace, shellOpts);
+    deleteNamespace(true, namespace, kubeconfig, shellOpts);
     werft.done(shellOpts.slice)
 }
 
-export async function wipeAndRecreateNamespace(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
-    await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, shellOpts);
+export async function wipeAndRecreateNamespace(helmInstallName: string, namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
+    await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, kubeconfig, shellOpts);
 
-    createNamespace(namespace, shellOpts);
+    createNamespace(namespace, kubeconfig, shellOpts);
 }
 
-export async function wipePreviewEnvironmentHelm(helmInstallName: string, namespace: string, shellOpts: ExecOptions) {
+export async function wipePreviewEnvironmentHelm(helmInstallName: string, namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
     // uninstall helm first so that:
     //  - ws-scaler can't create new ghosts in the meantime
     //  - ws-manager can't start new probes/workspaces
     uninstallHelm(helmInstallName, namespace, shellOpts)
 }
 
-async function wipePreviewEnvironmentInstaller(namespace: string, shellOpts: ExecOptions) {
+async function wipePreviewEnvironmentInstaller(namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
     const slice = shellOpts.slice || "installer";
     const werft = getGlobalWerftInstance();
 
-    const hasGitpodConfigmap = (exec(`kubectl -n ${namespace} get configmap gitpod-app`, { slice, dontCheckRc: true })).code === 0;
+    const hasGitpodConfigmap = (exec(`kubectl --kubeconfig ${kubeconfig} -n ${namespace} get configmap gitpod-app`, { slice, dontCheckRc: true })).code === 0;
     if (hasGitpodConfigmap) {
         werft.log(slice, `${namespace} has Gitpod configmap, proceeding with removal`);
         const inWerftFolder = exec(`pwd`, { slice, dontCheckRc: true }).stdout.trim().endsWith(".werft");
         if (inWerftFolder) {
             // used in .werft/wipe-devstaging.yaml on preview environment clean-up
-            exec(`./util/uninstall-gitpod.sh ${namespace}`, { slice });
+            exec(`./util/uninstall-gitpod.sh ${namespace} ${kubeconfig}`, { slice });
         } else {
             // used in .werft/build.yaml on 'with-clean-slate-deployment=true'
-            exec(`./.werft/util/uninstall-gitpod.sh ${namespace}`, { slice });
+            exec(`./.werft/util/uninstall-gitpod.sh ${namespace} ${kubeconfig}`, { slice });
         }
 
     } else {
@@ -80,8 +79,8 @@ function uninstallHelm(installationName: string, namespace: string, shellOpts: E
 }
 
 // Delete pods for running workspaces, even if they are stuck in terminating because of the finalizer decorator
-function deleteAllWorkspaces(namespace: string, shellOpts: ExecOptions) {
-    const objs = exec(`kubectl get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, async: false })
+function deleteAllWorkspaces(namespace: string, kubecofig: string, shellOpts: ExecOptions) {
+    const objs = exec(`kubectl --kubeconfig ${kubecofig} get pod -l component=workspace --namespace ${namespace} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, async: false })
         .split("\n")
         .map(o => o.trim())
         .filter(o => o.length > 0);
@@ -89,14 +88,14 @@ function deleteAllWorkspaces(namespace: string, shellOpts: ExecOptions) {
     objs.forEach(o => {
         try {
             // In most cases the calls below fails because the workspace is already gone. Ignore those cases, log others.
-            exec(`kubectl patch pod --namespace ${namespace} ${o} -p '{"metadata":{"finalizers":null}}'`, { ...shellOpts });
-            const result = exec(`kubectl delete pod --namespace ${namespace} ${o} --ignore-not-found=true --timeout=10s`, { ...shellOpts, async: false, dontCheckRc: true });
+            exec(`kubectl --kubeconfig ${kubecofig} patch pod --namespace ${namespace} ${o} -p '{"metadata":{"finalizers":null}}'`, { ...shellOpts });
+            const result = exec(`kubectl --kubeconfig ${kubecofig} delete pod --namespace ${namespace} ${o} --ignore-not-found=true --timeout=10s`, { ...shellOpts, async: false, dontCheckRc: true });
             if (result.code !== 0) {
                 // We hit a timeout, and have no clue why. Manually re-trying has shown to consistenly being not helpful, either. Thus use THE FORCE.
-                exec(`kubectl delete pod --namespace ${namespace} ${o} --ignore-not-found=true --force`, { ...shellOpts });
+                exec(`kubectl --kubeconfig ${kubecofig} delete pod --namespace ${namespace} ${o} --ignore-not-found=true --force`, { ...shellOpts });
             }
         } catch (err) {
-            const result = exec(`kubectl get pod --namespace ${namespace} ${o}`, { ...shellOpts, dontCheckRc: true, async: false });
+            const result = exec(`kubectl --kubeconfig ${kubecofig} get pod --namespace ${namespace} ${o}`, { ...shellOpts, dontCheckRc: true, async: false });
             if (result.code === 0) {
                 console.error(`unable to patch/delete ${o} but it's still on the dataplane`);
             }
@@ -105,14 +104,14 @@ function deleteAllWorkspaces(namespace: string, shellOpts: ExecOptions) {
 }
 
 // deleteAllUnnamespacedObjects deletes all unnamespaced objects for the given namespace
-async function deleteAllUnnamespacedObjects(namespace: string, shellOpts: ExecOptions): Promise<void> {
+async function deleteAllUnnamespacedObjects(namespace: string, kubeconfig: string, shellOpts: ExecOptions): Promise<void> {
     const werft = getGlobalWerftInstance()
     const slice = shellOpts.slice || "deleteobjs";
 
     const promisedDeletes: Promise<any>[] = [];
     for (const resType of ["clusterrole", "clusterrolebinding", "podsecuritypolicy"]) {
         werft.log(slice, `Searching and filtering ${resType}s...`);
-        const objs = exec(`kubectl get ${resType} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, slice, async: false })
+        const objs = exec(`kubectl --kubeconfig ${kubeconfig} get ${resType} --no-headers -o=custom-columns=:metadata.name`, { ...shellOpts, slice, async: false })
             .split("\n")
             .map(o => o.trim())
             .filter(o => o.length > 0)
@@ -120,14 +119,14 @@ async function deleteAllUnnamespacedObjects(namespace: string, shellOpts: ExecOp
 
         werft.log(slice, `Deleting old ${resType}s...`);
         for (const obj of objs) {
-            promisedDeletes.push(exec(`kubectl delete ${resType} ${obj}`, { ...shellOpts, slice, async: true }) as Promise<any>);
+            promisedDeletes.push(exec(`kubectl --kubeconfig ${kubeconfig} delete ${resType} ${obj}`, { ...shellOpts, slice, async: true }) as Promise<any>);
         }
     }
     await Promise.all(promisedDeletes);
 }
 
-export function createNamespace(namespace: string, shellOpts: ExecOptions) {
-    const result = (exec(`kubectl get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
+export function createNamespace(namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
+    const result = (exec(`kubectl --kubeconfig ${kubeconfig} get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
     const exists = result.code === 0;
     if (exists) {
         return;
@@ -135,43 +134,43 @@ export function createNamespace(namespace: string, shellOpts: ExecOptions) {
 
     // (re-)create namespace
     [
-        `kubectl create namespace ${namespace}`,
-        `kubectl patch namespace ${namespace} --patch '{"metadata": {"labels": {"${IS_PREVIEW_APP_LABEL}": "true"}}}'`
+        `kubectl --kubeconfig ${kubeconfig} create namespace ${namespace}`,
+        `kubectl --kubeconfig ${kubeconfig} patch namespace ${namespace} --patch '{"metadata": {"labels": {"${IS_PREVIEW_APP_LABEL}": "true"}}}'`
     ].forEach((cmd) => exec(cmd, shellOpts));
 };
 
-export function listAllPreviewNamespaces(shellOpts: ExecOptions): string[] {
-    return exec(`kubectl get namespaces -l ${IS_PREVIEW_APP_LABEL}=true -o=custom-columns=:metadata.name`, { ...shellOpts, silent: true, async: false })
+export function listAllPreviewNamespaces(kubeconfig: string, shellOpts: ExecOptions): string[] {
+    return exec(`kubectl --kubeconfig ${kubeconfig} get namespaces -l ${IS_PREVIEW_APP_LABEL}=true -o=custom-columns=:metadata.name`, { ...shellOpts, silent: true, async: false })
         .stdout
         .split("\n")
         .map(o => o.trim())
         .filter(o => o.length > 0);
 }
 
-export function deleteNamespace(wait: boolean, namespace: string, shellOpts: ExecOptions) {
+export function deleteNamespace(wait: boolean, namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
     // check if present
-    const result = (exec(`kubectl get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
+    const result = (exec(`kubectl --kubeconfig ${kubeconfig} get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
     if (result.code !== 0) {
         return;
     }
 
-    const cmd = `kubectl delete namespace ${namespace}`;
+    const cmd = `kubectl --kubeconfig ${kubeconfig} delete namespace ${namespace}`;
     exec(cmd, shellOpts);
 
     // wait until deletion was successful
     while (wait) {
-        const result = (exec(`kubectl get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
+        const result = (exec(`kubectl --kubeconfig ${kubeconfig} get namespace ${namespace}`, { ...shellOpts, dontCheckRc: true, async: false }));
         wait = result.code === 0;
     }
 }
 
-export async function deleteNonNamespaceObjects(namespace: string, destname: string, shellOpts: ExecOptions) {
-    exec(`/usr/local/bin/helm3 delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, { ...shellOpts });
+export async function deleteNonNamespaceObjects(namespace: string, destname: string, kubeconfig: string, shellOpts: ExecOptions) {
+    exec(`/usr/local/bin/helm3 --kubeconfig ${kubeconfig} delete gitpod-${destname} || echo gitpod-${destname} was not installed yet`, { ...shellOpts });
 
     let objs = [];
     ["node-daemon", "cluster", "workspace", "ws-sync", "ws-manager-node", "ws-daemon", "registry-facade"].forEach(comp =>
         ["ClusterRole", "ClusterRoleBinding", "PodSecurityPolicy"].forEach(kind =>
-            exec(`kubectl get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`, { ...shellOpts, dontCheckRc: true, async: false })
+            exec(`kubectl --kubeconfig ${kubeconfig} get ${kind} -l component=${comp} --no-headers -o=custom-columns=:metadata.name | grep ${namespace}-ns`, { ...shellOpts, dontCheckRc: true, async: false })
                 .split("\n")
                 .map(o => o.trim())
                 .filter(o => o.length > 0)
@@ -181,7 +180,7 @@ export async function deleteNonNamespaceObjects(namespace: string, destname: str
 
     const promisedDeletes: Promise<any>[] = [];
     objs.forEach(o => {
-        promisedDeletes.push(exec(`kubectl delete ${o.kind} ${o.obj}`, { ...shellOpts, async: true }) as Promise<any>);
+        promisedDeletes.push(exec(`kubectl --kubeconfig ${kubeconfig} delete ${o.kind} ${o.obj}`, { ...shellOpts, async: true }) as Promise<any>);
     });
     await Promise.all(promisedDeletes);
 }
@@ -191,21 +190,21 @@ export interface PortRange {
     end: number;
 }
 
-export function findLastHostPort(namespace: string, name: string, shellOpts: ExecOptions): number {
-    const portStr = exec(`kubectl get ds -n ${namespace} ${name} -o yaml | yq r - 'spec.template.spec.containers.*.ports.*.hostPort'`, { ...shellOpts, silent: true, async: false }).stdout
+export function findLastHostPort(namespace: string, name: string, kubeconfig: string, shellOpts: ExecOptions): number {
+    const portStr = exec(`kubectl --kubeconfig ${kubeconfig} get ds -n ${namespace} ${name} -o yaml | yq r - 'spec.template.spec.containers.*.ports.*.hostPort'`, { ...shellOpts, silent: true, async: false }).stdout
     return Number.parseInt(portStr)
 }
 
-export function findFreeHostPorts(ranges: PortRange[], shellOpts: ExecOptions): number[] {
+export function findFreeHostPorts(ranges: PortRange[], kubeconfig: string, shellOpts: ExecOptions): number[] {
     const werft = getGlobalWerftInstance()
-    const hostPorts: number[] = exec(`kubectl get pods --all-namespaces -o yaml | yq r - 'items.*.spec.containers.*.ports.*.hostPort'`, { ...shellOpts, silent: true, async: false })
+    const hostPorts: number[] = exec(`kubectl --kubeconfig ${kubeconfig} get pods --all-namespaces -o yaml | yq r - 'items.*.spec.containers.*.ports.*.hostPort'`, { ...shellOpts, silent: true, async: false })
         .stdout
         .split("\n")
         .map(l => l.trim())
         .filter(l => l.length > 0)
         .map(l => Number.parseInt(l));
 
-    const nodePorts: number[] = exec(`kubectl get services --all-namespaces -o yaml | yq r - 'items.*.spec.ports.*.nodePort'`, { ...shellOpts, silent: true, async: false })
+    const nodePorts: number[] = exec(`kubectl --kubeconfig ${kubeconfig} get services --all-namespaces -o yaml | yq r - 'items.*.spec.ports.*.nodePort'`, { ...shellOpts, silent: true, async: false })
         .stdout
         .split("\n")
         .map(l => l.trim())
@@ -238,8 +237,8 @@ export function findFreeHostPorts(ranges: PortRange[], shellOpts: ExecOptions):
     return results;
 }
 
-export function waitForDeploymentToSucceed(name: string, namespace: string, type: string, shellOpts: ExecOptions) {
-    exec(`kubectl rollout status ${type} ${name} -n ${namespace}`, shellOpts);
+export function waitForDeploymentToSucceed(name: string, namespace: string, type: string, kubeconfig: string, shellOpts: ExecOptions) {
+    exec(`kubectl --kubeconfig ${kubeconfig} rollout status ${type} ${name} -n ${namespace}`, shellOpts);
 }
 
 interface Pod {
@@ -248,13 +247,13 @@ interface Pod {
     phase: string
 }
 
-export async function waitUntilAllPodsAreReady(namespace: string, shellOpts: ExecOptions) {
+export async function waitUntilAllPodsAreReady(namespace: string, kubeconfig: string, shellOpts: ExecOptions) {
     const werft = getGlobalWerftInstance();
     werft.log(shellOpts.slice, `Waiting until all pods in namespace ${namespace} are Running/Succeeded/Completed.`)
     for (let i = 0; i < 200; i++) {
         let pods: Pod[]
         try {
-            pods = getPods(namespace)
+            pods = getPods(namespace, kubeconfig)
         } catch (err) {
             werft.log(shellOpts.slice, err)
             continue
@@ -280,12 +279,12 @@ export async function waitUntilAllPodsAreReady(namespace: string, shellOpts: Exe
 
         await sleep(3 * 1000)
     }
-    exec(`kubectl get pods -n ${namespace}`, { ...shellOpts, async: false })
+    exec(`kubectl --kubeconfig ${kubeconfig} get pods -n ${namespace}`, { ...shellOpts, async: false })
     throw new Error(`Not all pods in namespace ${namespace} transitioned to 'Running' or 'Succeeded/Completed' during the expected time.`)
 }
 
-function getPods(namespace: string): Pod[] {
-    const cmd = `kubectl get pods -n ${namespace}  -o=jsonpath='{range .items[*]}{@.metadata.name}:{@.metadata.ownerReferences[0].kind}:{@.status.phase};{end}'`
+function getPods(namespace: string, kubeconfig: string): Pod[] {
+    const cmd = `kubectl --kubeconfig ${kubeconfig} get pods -n ${namespace}  -o=jsonpath='{range .items[*]}{@.metadata.name}:{@.metadata.ownerReferences[0].kind}:{@.status.phase};{end}'`
     const unsanitizedPods = exec(cmd, { silent: true, async: false, dontCheckRc: true });
     if (unsanitizedPods.code != 0) {
         throw new Error(`"${cmd}" failed with code ${unsanitizedPods.code}; stdout: ${unsanitizedPods.stdout}; stderr: ${unsanitizedPods.stderr}`)
@@ -298,11 +297,11 @@ function getPods(namespace: string): Pod[] {
         .map(s => { const i = s.split(":"); return { name: i[0], owner: i[1], phase: i[2] } })
 }
 
-export async function waitForApiserver(shellOpts: ExecOptions) {
+export async function waitForApiserver(kubeconfig: string, shellOpts: ExecOptions) {
     const werft = getGlobalWerftInstance();
     for (let i = 0; i < 300; i++) {
         werft.log(shellOpts.slice, 'Checking that k3s apiserver is ready...')
-        const result = exec(`kubectl get --raw='/readyz?verbose'`, { ...shellOpts, dontCheckRc: true, async: false });
+        const result = exec(`kubectl --kubeconfig ${kubeconfig} get --raw='/readyz?verbose'`, { ...shellOpts, dontCheckRc: true, async: false });
         if (result.code == 0) {
             werft.log(shellOpts.slice, 'k3s apiserver is ready')
             return;
diff --git a/.werft/util/uninstall-gitpod.sh b/.werft/util/uninstall-gitpod.sh
index eeaa30cfbfe4cd..2a1ef1837b707d 100755
--- a/.werft/util/uninstall-gitpod.sh
+++ b/.werft/util/uninstall-gitpod.sh
@@ -3,6 +3,7 @@
 set -euo pipefail
 
 NAMESPACE=$1
+KUBECONFIG=$2
 
 if [[ -z ${NAMESPACE} ]]; then
    echo "One or more input params were invalid. The params we received were: ${NAMESPACE}"
@@ -10,11 +11,11 @@ if [[ -z ${NAMESPACE} ]]; then
 fi
 
 echo "Removing Gitpod in namespace ${NAMESPACE}"
-kubectl get configmap gitpod-app -n "${NAMESPACE}" -o jsonpath='{.data.app\.yaml}' | kubectl delete --ignore-not-found=true -f -
+kubectl --kubeconfig "$KUBECONFIG" get configmap gitpod-app -n "${NAMESPACE}" -o jsonpath='{.data.app\.yaml}' | kubectl --kubeconfig "$KUBECONFIG" delete --ignore-not-found=true -f -
 
 echo "Removing Gitpod storage from ${NAMESPACE}"
-kubectl -n "${NAMESPACE}" --ignore-not-found=true delete pvc data-mysql-0
+kubectl --kubeconfig "$KUBECONFIG" -n "${NAMESPACE}" --ignore-not-found=true delete pvc data-mysql-0
 # the installer includes the minio PVC in it's config mpap, this is a "just in case"
-kubectl -n "${NAMESPACE}" delete pvc minio || true
+kubectl --kubeconfig "$KUBECONFIG" -n "${NAMESPACE}" delete pvc minio || true
 
 echo "Successfully removed Gitpod from ${NAMESPACE}"
\ No newline at end of file
diff --git a/.werft/vm/vm.ts b/.werft/vm/vm.ts
index 31d20bbf21afd2..82ef329fa5e9cf 100644
--- a/.werft/vm/vm.ts
+++ b/.werft/vm/vm.ts
@@ -19,7 +19,7 @@ EOF
 /**
  * Convenience function to kubectl delete a manifest from stdin.
  */
- function kubectlDeleteManifest(manifest: string, options?: { validate?: boolean }) {
+function kubectlDeleteManifest(manifest: string, options?: { validate?: boolean }) {
     exec(`
         cat <<EOF | kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} delete -f -
 ${manifest}
@@ -71,7 +71,7 @@ export function startVM(options: { name: string }) {
 /**
  * Remove a VM with its Namespace
  */
- export function deleteVM(options: { name: string }) {
+export function deleteVM(options: { name: string }) {
     const namespace = `preview-${options.name}`
     const userDataSecretName = `userdata-${options.name}`
 
@@ -182,9 +182,9 @@ export function stopKubectlPortForwards() {
 /**
  * Install Fluent-Bit sending logs to GCP
  */
-export function installFluentBit(options: {namespace: string, slice: string}) {
-    exec(`kubectl create secret generic fluent-bit-external --save-config --dry-run=client --from-file=credentials.json=/mnt/fluent-bit-external/credentials.json -o yaml | kubectl apply -n ${options.namespace} -f -`, { slice: options.slice, dontCheckRc: true})
-    exec(`helm3 repo add fluent https://fluent.github.io/helm-charts`, { slice: options.slice, dontCheckRc: true})
-    exec(`helm3 repo update`, { slice: options.slice, dontCheckRc: true})
-    exec(`helm3 upgrade --install fluent-bit fluent/fluent-bit -n ${options.namespace} -f .werft/vm/charts/fluentbit/values.yaml`, { slice: options.slice, dontCheckRc: true})
+export function installFluentBit(options: { namespace: string, kubeconfig: string, slice: string }) {
+    exec(`kubectl --kubeconfig ${options.kubeconfig} create secret generic fluent-bit-external --save-config --dry-run=client --from-file=credentials.json=/mnt/fluent-bit-external/credentials.json -o yaml | kubectl --kubeconfig ${options.kubeconfig} apply -n ${options.namespace} -f -`, { slice: options.slice, dontCheckRc: true })
+    exec(`helm3 --kubeconfig ${options.kubeconfig} repo add fluent https://fluent.github.io/helm-charts`, { slice: options.slice, dontCheckRc: true })
+    exec(`helm3 --kubeconfig ${options.kubeconfig} repo update`, { slice: options.slice, dontCheckRc: true })
+    exec(`helm3 --kubeconfig ${options.kubeconfig} upgrade --install fluent-bit fluent/fluent-bit -n ${options.namespace} -f .werft/vm/charts/fluentbit/values.yaml`, { slice: options.slice, dontCheckRc: true })
 }
\ No newline at end of file

From ea745b617f1c1a3a94abb1b817324f2e801eb883 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Wed, 30 Mar 2022 10:04:21 +0000
Subject: [PATCH 3/9] Fix kubeconfig on cleanup jobs

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/platform-delete-preview-environments-cron.ts | 7 ++++---
 .werft/wipe-devstaging.ts                           | 7 +++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.werft/platform-delete-preview-environments-cron.ts b/.werft/platform-delete-preview-environments-cron.ts
index 72e16bfef9d0e8..eb974363930cb4 100644
--- a/.werft/platform-delete-preview-environments-cron.ts
+++ b/.werft/platform-delete-preview-environments-cron.ts
@@ -4,6 +4,7 @@ import { SpanStatusCode } from '@opentelemetry/api';
 import { wipePreviewEnvironmentAndNamespace, helmInstallName, listAllPreviewNamespaces } from './util/kubectl';
 import { exec } from './util/shell';
 import { previewNameFromBranchName } from './util/preview';
+import { CORE_DEV_KUBECONFIG_PATH } from './jobs/build/const';
 
 // Will be set once tracing has been initialized
 let werft: Werft
@@ -30,7 +31,7 @@ async function deletePreviewEnvironments() {
     try {
         const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
         exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`);
-        exec('gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev');
+        exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`);
     } catch (err) {
         werft.fail("prep", err)
     }
@@ -46,7 +47,7 @@ async function deletePreviewEnvironments() {
     werft.phase("Fetching previews");
     let previews: string[]
     try {
-        previews = listAllPreviewNamespaces({});
+        previews = listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, {});
         previews.forEach(previewNs => werft.log("Fetching previews", previewNs))
     } catch (err) {
         werft.fail("Fetching previews", err)
@@ -57,7 +58,7 @@ async function deletePreviewEnvironments() {
     try {
         const previewsToDelete = previews.filter(ns => !expectedPreviewEnvironmentNamespaces.has(ns))
         // Trigger namespace deletion in parallel
-        const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, { slice: `Deleting preview ${preview}` }));
+        const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, CORE_DEV_KUBECONFIG_PATH, { slice: `Deleting preview ${preview}` }));
         // But wait for all of them to finish before (or one of them to fail) before we continue
         await Promise.all(promises)
     } catch (err) {
diff --git a/.werft/wipe-devstaging.ts b/.werft/wipe-devstaging.ts
index 2245081be286e0..871776c468e877 100644
--- a/.werft/wipe-devstaging.ts
+++ b/.werft/wipe-devstaging.ts
@@ -1,11 +1,10 @@
 import { Werft } from './util/werft'
 import { wipePreviewEnvironmentAndNamespace, listAllPreviewNamespaces, helmInstallName } from './util/kubectl';
-import * as fs from 'fs';
-import { deleteExternalIp } from './util/gcloud';
 import * as Tracing from './observability/tracing'
 import { SpanStatusCode } from '@opentelemetry/api';
 import { ExecOptions } from './util/shell';
 import { env } from './util/util';
+import { CORE_DEV_KUBECONFIG_PATH } from './jobs/build/const';
 
 // Will be set once tracing has been initialized
 let werft: Werft
@@ -15,7 +14,7 @@ async function wipePreviewCluster(shellOpts: ExecOptions) {
     const namespaces: string[] = [];
     if (namespace_raw === "<no value>" || !namespace_raw) {
         werft.log('wipe', "Going to wipe all namespaces");
-        listAllPreviewNamespaces(shellOpts)
+        listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, shellOpts)
             .map(ns => namespaces.push(ns));
     } else {
         werft.log('wipe', `Going to wipe namespace ${namespace_raw}`);
@@ -23,7 +22,7 @@ async function wipePreviewCluster(shellOpts: ExecOptions) {
     }
 
     for (const namespace of namespaces) {
-        await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, { ...shellOpts, slice: 'wipe' });
+        await wipePreviewEnvironmentAndNamespace(helmInstallName, namespace, CORE_DEV_KUBECONFIG_PATH, { ...shellOpts, slice: 'wipe' });
     }
 }
 

From eb173bdf3d16df0eb26062e87b6d7ac003fe18d2 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Wed, 30 Mar 2022 10:34:15 +0000
Subject: [PATCH 4/9] Fix certification issuing and installation

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/deploy-to-preview-environment.ts | 7 ++++---
 .werft/util/certs.ts                               | 5 +++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index 836c8272c37ae3..ec5d51c32d06f4 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -239,7 +239,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
 
             // trigger certificate issuing
             await issueMetaCerts(werft, namespace, "certs", domain, withVM);
-            await installMetaCertificates(werft, namespace);
+            await installMetaCertificates(werft, namespace, CORE_DEV_KUBECONFIG_PATH);
             werft.done(installerSlices.ISSUE_CERTIFICATES);
         } catch (err) {
             if (!jobConfig.mainBuild) {
@@ -388,7 +388,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         // trigger certificate issuing
         werft.log('certificate', "organizing a certificate for the preview environment...");
         await issueMetaCerts(werft, namespace, "certs", domain, false);
-        await installMetaCertificates(werft, namespace);
+        await installMetaCertificates(werft, namespace, CORE_DEV_KUBECONFIG_PATH);
         werft.done('certificate');
         await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, false, CORE_DEV_KUBECONFIG_PATH)
         werft.done('prep');
@@ -688,13 +688,14 @@ export async function issueMetaCerts(werft: Werft, previewNamespace: string, cer
     await issueCertficate(werft, metaClusterCertParams, metaEnv());
 }
 
-async function installMetaCertificates(werft: Werft, namespace: string) {
+async function installMetaCertificates(werft: Werft, namespace: string, kubeconfig: string) {
     const certName = namespace;
     const metaInstallCertParams = new InstallCertificateParams()
     metaInstallCertParams.certName = certName
     metaInstallCertParams.certNamespace = "certs"
     metaInstallCertParams.certSecretName = PROXY_SECRET_NAME
     metaInstallCertParams.destinationNamespace = namespace
+    metaInstallCertParams.destinationKubeconfig = kubeconfig
     await installCertficate(werft, metaInstallCertParams, metaEnv());
 }
 
diff --git a/.werft/util/certs.ts b/.werft/util/certs.ts
index 35c4b78e409a06..75c50b8c526c5c 100644
--- a/.werft/util/certs.ts
+++ b/.werft/util/certs.ts
@@ -21,6 +21,7 @@ export class InstallCertificateParams {
     certSecretName: string
     certNamespace: string
     destinationNamespace: string
+    destinationKubeconfig: string
 }
 
 export async function issueCertficate(werft, params: IssueCertificateParams, shellOpts: ExecOptions) {
@@ -71,13 +72,13 @@ export async function issueCertficate(werft, params: IssueCertificateParams, she
 export async function installCertficate(werft, params: InstallCertificateParams, shellOpts: ExecOptions) {
     let notReadyYet = true;
     werft.log('certificate', `copying certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
-    const cmd = `kubectl get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
+    const cmd = `kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
     | yq d - 'metadata.namespace' \
     | yq d - 'metadata.uid' \
     | yq d - 'metadata.resourceVersion' \
     | yq d - 'metadata.creationTimestamp' \
     | sed 's/${params.certName}/${params.certSecretName}/g' \
-    | kubectl apply --namespace=${params.destinationNamespace} -f -`
+    | kubectl --kubeconfig ${params.destinationKubeconfig} apply --namespace=${params.destinationNamespace} -f -`
 
     for (let i = 0; i < 60 && notReadyYet; i++) {
         const result = exec(cmd, { ...shellOpts, silent: true, dontCheckRc: true, async: false });

From c649382a8125760252ebf6ec22e2b0b3c8e4dbc0 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Thu, 31 Mar 2022 08:08:11 +0000
Subject: [PATCH 5/9] Fix observability deployment

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/deploy-to-preview-environment.ts | 1 -
 .werft/observability/monitoring-satellite.ts       | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index ec5d51c32d06f4..fd9aba8eac0642 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -703,7 +703,6 @@ async function installMonitoring(kubeconfig: string, namespace: string, nodeExpo
     const installMonitoringSatelliteParams = new InstallMonitoringSatelliteParams();
     installMonitoringSatelliteParams.kubeconfigPath = kubeconfig
     installMonitoringSatelliteParams.branch = observabilityBranch;
-    installMonitoringSatelliteParams.kubeconfigPath = ""
     installMonitoringSatelliteParams.satelliteNamespace = namespace
     installMonitoringSatelliteParams.clusterName = namespace
     installMonitoringSatelliteParams.nodeExporterPort = nodeExporterPort
diff --git a/.werft/observability/monitoring-satellite.ts b/.werft/observability/monitoring-satellite.ts
index d764118c5de255..172d6a725a6db3 100644
--- a/.werft/observability/monitoring-satellite.ts
+++ b/.werft/observability/monitoring-satellite.ts
@@ -83,7 +83,7 @@ async function ensureCorrectInstallationOrder(kubeconfig: string, namespace: str
     const werft = getGlobalWerftInstance()
 
     werft.log(sliceName, 'installing monitoring-satellite')
-    exec(`cd observability && KUBECONFIG=${kubeconfig} hack/deploy-satellite.sh`, {slice: sliceName})
+    exec(`cd observability && hack/deploy-satellite.sh --kubeconfig ${kubeconfig}`, {slice: sliceName})
 
     deployGitpodServiceMonitors(kubeconfig)
     checkReadiness(kubeconfig, namespace, checkNodeExporterStatus)
@@ -92,7 +92,7 @@ async function ensureCorrectInstallationOrder(kubeconfig: string, namespace: str
 async function checkReadiness(kubeconfig: string, namespace: string, checkNodeExporterStatus: boolean) {
     // For some reason prometheus' statefulset always take quite some time to get created
     // Therefore we wait a couple of seconds
-    exec(`sleep 30 && kubectl rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
+    exec(`sleep 30 && kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
     exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
     exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
     exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})

From bd1d2f24eadd6f67422b82a413783a6336f6e115 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Thu, 31 Mar 2022 09:00:43 +0000
Subject: [PATCH 6/9] Fix certificates on Harvester previews

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .../build/deploy-to-preview-environment.ts    | 23 ++++++++++---------
 .werft/util/certs.ts                          |  4 ++--
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index fd9aba8eac0642..837ea1bbd77eda 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -4,13 +4,14 @@ import * as fs from 'fs';
 import { exec, ExecOptions } from '../../util/shell';
 import { InstallMonitoringSatelliteParams, installMonitoringSatellite } from '../../observability/monitoring-satellite';
 import { wipeAndRecreateNamespace, setKubectlContextNamespace, deleteNonNamespaceObjects, findFreeHostPorts, createNamespace, helmInstallName, findLastHostPort, waitUntilAllPodsAreReady, waitForApiserver } from '../../util/kubectl';
-import { issueCertficate, installCertficate, IssueCertificateParams, InstallCertificateParams } from '../../util/certs';
+import { issueCertificate, installCertificate, IssueCertificateParams, InstallCertificateParams } from '../../util/certs';
 import { sleep, env } from '../../util/util';
 import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, PREVIEW_K3S_KUBECONFIG_PATH } from "./const";
 import { Werft } from "../../util/werft";
 import { JobConfig } from "./job-config";
 import * as VM from '../../vm/vm'
 import { Analytics, Installer } from "./installer/installer";
+import { previewNameFromBranchName } from "../../util/preview";
 
 // used by both deploys (helm and Installer)
 const PROXY_SECRET_NAME = "proxy-config-certificates";
@@ -141,7 +142,8 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         werft.done(vmSlices.EXTERNAL_LOGGING)
 
 
-        issueMetaCerts(werft, PROXY_SECRET_NAME, "default", domain, withVM)
+        issueMetaCerts(werft, PROXY_SECRET_NAME, "certs", domain, withVM)
+        installMetaCertificates(werft, jobConfig.repository.branch, "default", PREVIEW_K3S_KUBECONFIG_PATH)
         werft.done('certificate')
         installMonitoring(PREVIEW_K3S_KUBECONFIG_PATH, deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch);
         werft.done('observability')
@@ -239,7 +241,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
 
             // trigger certificate issuing
             await issueMetaCerts(werft, namespace, "certs", domain, withVM);
-            await installMetaCertificates(werft, namespace, CORE_DEV_KUBECONFIG_PATH);
+            await installMetaCertificates(werft, jobConfig.repository.branch, namespace, CORE_DEV_KUBECONFIG_PATH);
             werft.done(installerSlices.ISSUE_CERTIFICATES);
         } catch (err) {
             if (!jobConfig.mainBuild) {
@@ -388,7 +390,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
         // trigger certificate issuing
         werft.log('certificate', "organizing a certificate for the preview environment...");
         await issueMetaCerts(werft, namespace, "certs", domain, false);
-        await installMetaCertificates(werft, namespace, CORE_DEV_KUBECONFIG_PATH);
+        await installMetaCertificates(werft, jobConfig.repository.branch, namespace, CORE_DEV_KUBECONFIG_PATH);
         werft.done('certificate');
         await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, false, CORE_DEV_KUBECONFIG_PATH)
         werft.done('prep');
@@ -448,7 +450,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
     werft.log(`observability`, "Installing monitoring-satellite...")
     if (deploymentConfig.withObservability) {
         try {
-            await installMonitoring(CORE_DEV_KUBECONFIG_PATH ,namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, false, jobConfig.observability.branch);
+            await installMonitoring(CORE_DEV_KUBECONFIG_PATH, namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, false, jobConfig.observability.branch);
         } catch (err) {
             if (!jobConfig.mainBuild) {
                 werft.fail('observability', err);
@@ -685,18 +687,17 @@ export async function issueMetaCerts(werft: Werft, previewNamespace: string, cer
     metaClusterCertParams.ip = getCoreDevIngressIP();
     metaClusterCertParams.bucketPrefixTail = ""
     metaClusterCertParams.additionalSubdomains = additionalSubdomains
-    await issueCertficate(werft, metaClusterCertParams, metaEnv());
+    await issueCertificate(werft, metaClusterCertParams, metaEnv());
 }
 
-async function installMetaCertificates(werft: Werft, namespace: string, kubeconfig: string) {
-    const certName = namespace;
+async function installMetaCertificates(werft: Werft, branch: string, destNamespace: string, kubeconfig: string) {
     const metaInstallCertParams = new InstallCertificateParams()
-    metaInstallCertParams.certName = certName
+    metaInstallCertParams.certName = `staging-${previewNameFromBranchName(branch)}`;
     metaInstallCertParams.certNamespace = "certs"
     metaInstallCertParams.certSecretName = PROXY_SECRET_NAME
-    metaInstallCertParams.destinationNamespace = namespace
+    metaInstallCertParams.destinationNamespace = destNamespace
     metaInstallCertParams.destinationKubeconfig = kubeconfig
-    await installCertficate(werft, metaInstallCertParams, metaEnv());
+    await installCertificate(werft, metaInstallCertParams, metaEnv());
 }
 
 async function installMonitoring(kubeconfig: string, namespace: string, nodeExporterPort: number, domain: string, stackdriverServiceAccount: any, withVM: boolean, observabilityBranch: string) {
diff --git a/.werft/util/certs.ts b/.werft/util/certs.ts
index 75c50b8c526c5c..be28c82c32d0a4 100644
--- a/.werft/util/certs.ts
+++ b/.werft/util/certs.ts
@@ -24,7 +24,7 @@ export class InstallCertificateParams {
     destinationKubeconfig: string
 }
 
-export async function issueCertficate(werft, params: IssueCertificateParams, shellOpts: ExecOptions) {
+export async function issueCertificate(werft, params: IssueCertificateParams, shellOpts: ExecOptions) {
     var subdomains = [];
     werft.log("certificate", `Subdomains: ${params.additionalSubdomains}`)
     for (const sd of params.additionalSubdomains) {
@@ -69,7 +69,7 @@ export async function issueCertficate(werft, params: IssueCertificateParams, she
 }
 
 
-export async function installCertficate(werft, params: InstallCertificateParams, shellOpts: ExecOptions) {
+export async function installCertificate(werft, params: InstallCertificateParams, shellOpts: ExecOptions) {
     let notReadyYet = true;
     werft.log('certificate', `copying certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
     const cmd = `kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \

From 21d96387126499482fdfa5eb333c2ba101731e51 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Thu, 31 Mar 2022 10:28:57 +0000
Subject: [PATCH 7/9] Fix helm-installer deployment iteration

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/deploy-to-preview-environment.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index 837ea1bbd77eda..f572f76c30e7e4 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -151,8 +151,9 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
 
     werft.phase(phases.PREDEPLOY, "Checking for existing installations...");
     // the context namespace is not set at this point
-    const hasGitpodHelmInstall = exec(`helm status ${helmInstallName} -n ${deploymentConfig.namespace}`, { slice: "check for Helm install", dontCheckRc: true }).code === 0;
-    const hasGitpodInstallerInstall = exec(`kubectl get configmap gitpod-app -n ${deploymentConfig.namespace}`, { slice: "check for Installer install", dontCheckRc: true }).code === 0;
+    const deploymentKubeconfig = withVM ? PREVIEW_K3S_KUBECONFIG_PATH : CORE_DEV_KUBECONFIG_PATH;
+    const hasGitpodHelmInstall = exec(`helm --kubeconfig ${deploymentKubeconfig} status ${helmInstallName} -n ${deploymentConfig.namespace}`, { slice: "check for Helm install", dontCheckRc: true }).code === 0;
+    const hasGitpodInstallerInstall = exec(`kubectl --kubeconfig ${deploymentKubeconfig} get configmap gitpod-app -n ${deploymentConfig.namespace}`, { slice: "check for Installer install", dontCheckRc: true }).code === 0;
     werft.log("result of installation checks", `has Helm install: ${hasGitpodHelmInstall}, has Installer install: ${hasGitpodInstallerInstall}`);
 
     if (withHelm) {

From 71ec9abe26560a15de43f1502359c8bcf27f2444 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Thu, 31 Mar 2022 10:53:26 +0000
Subject: [PATCH 8/9] Fix clean-slate-deployment

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/deploy-to-preview-environment.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.werft/jobs/build/deploy-to-preview-environment.ts b/.werft/jobs/build/deploy-to-preview-environment.ts
index f572f76c30e7e4..2ed7b0f482c5b4 100644
--- a/.werft/jobs/build/deploy-to-preview-environment.ts
+++ b/.werft/jobs/build/deploy-to-preview-environment.ts
@@ -218,7 +218,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
         if (deploymentConfig.cleanSlateDeployment && !withVM) {
             werft.log(installerSlices.CLEAN_ENV_STATE, "Clean the preview environment slate...");
             // re-create namespace
-            await cleanStateEnv(metaEnv());
+            await cleanStateEnv(deploymentKubeconfig, metaEnv());
 
         } else {
             werft.log(installerSlices.CLEAN_ENV_STATE, "Clean the preview environment slate...");
@@ -342,12 +342,12 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
 
     werft.done(phases.DEPLOY);
 
-    async function cleanStateEnv(shellOpts: ExecOptions) {
-        await wipeAndRecreateNamespace(helmInstallName, namespace, installer.options.kubeconfigPath, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
+    async function cleanStateEnv(kubeconfig: string, shellOpts: ExecOptions) {
+        await wipeAndRecreateNamespace(helmInstallName, namespace, kubeconfig, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
         // cleanup non-namespace objects
         werft.log(installerSlices.CLEAN_ENV_STATE, "removing old unnamespaced objects - this might take a while");
         try {
-            await deleteNonNamespaceObjects(namespace, destname, installer.options.kubeconfigPath, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
+            await deleteNonNamespaceObjects(namespace, destname, kubeconfig, { ...shellOpts, slice: installerSlices.CLEAN_ENV_STATE });
             werft.done(installerSlices.CLEAN_ENV_STATE);
         } catch (err) {
             werft.fail(installerSlices.CLEAN_ENV_STATE, err);

From 540ada2bf3809bdc9bd6d1e9ddbce83093b21e08 Mon Sep 17 00:00:00 2001
From: ArthurSens <arthursens2005@gmail.com>
Date: Thu, 31 Mar 2022 15:41:32 +0000
Subject: [PATCH 9/9] .werft/build/prepare: Split error throwing when
 configuring cluster credentials

Signed-off-by: ArthurSens <arthursens2005@gmail.com>
---
 .werft/jobs/build/prepare.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.werft/jobs/build/prepare.ts b/.werft/jobs/build/prepare.ts
index 6df1d7f3b7274d..22bde0e8aa1a17 100644
--- a/.werft/jobs/build/prepare.ts
+++ b/.werft/jobs/build/prepare.ts
@@ -56,10 +56,14 @@ function configureDocker() {
 
 function configureStaticClustersAccess() {
     const rcCoreDev = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
+    if (rcCoreDev != 0) {
+        throw new Error("Failed to get core-dev kubeconfig credentials.")
+    }
+
     const rcHarvester = exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
 
-    if (rcCoreDev != 0 || rcHarvester != 0) {
-        throw new Error("Failed to get core-dev kubeconfig credentials.")
+    if (rcHarvester != 0) {
+        throw new Error("Failed to get Harvester kubeconfig credentials.")
     }
 }