[werft] Add sweeper cleanup logic for k3s ws cluster (#4746)

* Update sweeper logic to delete k3s ws preview env and refactor methods and files

Author: Prince Rachit Sinha

.werft/util/gcloud.ts

@@ -0,0 +1,32 @@
+import { werft, exec } from './shell';
+import { sleep } from './util';
+
+export async function deleteExternalIp(phase: string, name: string, region = "europe-west1") {
+    const ip = getExternalIp(name)
+    werft.log(phase, `address describe returned: ${ip}`)
+    if (ip.indexOf("ERROR:") != -1 || ip == "") {
+        werft.log(phase, `no external static IP with matching name ${name} found`)
+        return
+    }
+
+    werft.log(phase, `found external static IP with matching name ${name}, will delete it`)
+    const cmd = `gcloud compute addresses delete ${name} --region ${region} --quiet`
+    let attempt = 0;
+    for (attempt = 0; attempt < 10; attempt++) {
+        let result = exec(cmd);
+        if (result.code === 0 && result.stdout.indexOf("Error") == -1) {
+            werft.log(phase, `external ip with name ${name} and ip ${ip} deleted`);
+            break;
+        } else {
+            werft.log(phase, `external ip with name ${name} and ip ${ip} could not be deleted, will reattempt`)
+        }
+        await sleep(5000)
+    }
+    if (attempt == 10) {
+        werft.log(phase, `could not delete the external ip with name ${name} and ip ${ip}`)
+    }
+}
+
+function getExternalIp(name: string, region = "europe-west1") {
+    return exec(`gcloud compute addresses describe ${name} --region ${region}| grep 'address:' | cut -c 10-`, { silent: true }).trim();
+}

.werft/wipe-devstaging.ts

@@ -1,23 +1,55 @@
-import { werft } from './util/shell';
+import { werft, exec } from './util/shell';
 import { wipePreviewEnvironment, listAllPreviewNamespaces } from './util/kubectl';
+import * as fs from 'fs';
+import { deleteExternalIp } from './util/gcloud';
 
 
-async function wipeDevstaging() {
+async function wipePreviewCluster(pathToKubeConfig: string) {
     const namespace_raw = process.env.NAMESPACE;
     const namespaces: string[] = [];
     if (namespace_raw === "<no value>" || !namespace_raw) {
         werft.log('wipe', "Going to wipe all namespaces");
-        listAllPreviewNamespaces("")
+        listAllPreviewNamespaces(pathToKubeConfig)
             .map(ns => namespaces.push(ns));
     } else {
         werft.log('wipe', `Going to wipe namespace ${namespace_raw}`);
         namespaces.push(namespace_raw);
     }
 
     for (const namespace of namespaces) {
-        await wipePreviewEnvironment("", "gitpod", namespace, { slice: 'wipe' });
+        await wipePreviewEnvironment(pathToKubeConfig, "gitpod", namespace, { slice: 'wipe' });
     }
-    werft.done('wipe');
 }
 
-wipeDevstaging()
+// if we have "/workspace/k3s-external.yaml" present that means a k3s ws cluster
+// exists, therefore, delete corresponding preview deployment from that cluster too
+// NOTE: Even for a non k3s ws deployment we will attempt to clean the preview.
+// This saves us from writing complex logic of querying meta cluster for registered workspaces
+// Since we use the same namespace to deploy in both dev and k3s cluster, this is safe
+async function k3sCleanup() {
+    if (fs.existsSync("/workspace/k3s-external.yaml")) {
+        werft.log("wipe", "found /workspace/k3s-external.yaml, assuming k3s ws cluster deployment exists, will attempt to wipe it")
+        await wipePreviewCluster("/workspace/k3s-external.yaml")
+        const namespace_raw = process.env.NAMESPACE;
+
+        // Since werft creates static external IP for ws-proxy of k3s using gcloud
+        // we delete it here. We retry because the ws-proxy-service which binds to this IP might not be deleted immediately
+        const k3sWsProxyIP =
+            deleteExternalIp("wipe", namespace_raw)
+    } else {
+        werft.log("wipe", `file /workspace/k3s-external.yaml does not exist, no cleanup for k3s cluster`)
+    }
+}
+
+// clean up the dev cluster in gitpod-core-dev
+async function devCleanup() {
+    await wipePreviewCluster("")
+}
+
+// sweeper runs in the dev cluster so we need to delete the k3s cluster first and then delete self contained namespace
+k3sCleanup().then(() => {
+    devCleanup()
+})
+
+
+werft.done('wipe');

.werft/wipe-devstaging.yaml

@@ -34,6 +34,7 @@ pod:
 
       export NAMESPACE="{{ .Annotations.namespace }}"
       sudo chown -R gitpod:gitpod /workspace
+      kubectl get secret k3sdev -n werft -ojsonpath='{.data}' | jq -r .[] | base64 -d > /workspace/k3s-external.yaml
 
       npm install shelljs semver ts-node typescript @types/shelljs @types/node @types/semver
       npx ts-node .werft/wipe-devstaging.ts