diff --git a/.werft/util/gcloud.ts b/.werft/util/gcloud.ts new file mode 100644 index 00000000000000..cbd88dee6985bc --- /dev/null +++ b/.werft/util/gcloud.ts @@ -0,0 +1,32 @@ +import { werft, exec } from './shell'; +import { sleep } from './util'; + +export async function deleteExternalIp(phase: string, name: string, region = "europe-west1") { + const ip = getExternalIp(name) + werft.log(phase, `address describe returned: ${ip}`) + if (ip.indexOf("ERROR:") != -1 || ip == "") { + werft.log(phase, `no external static IP with matching name ${name} found`) + return + } + + werft.log(phase, `found external static IP with matching name ${name}, will delete it`) + const cmd = `gcloud compute addresses delete ${name} --region ${region} --quiet` + let attempt = 0; + for (attempt = 0; attempt < 10; attempt++) { + let result = exec(cmd); + if (result.code === 0 && result.stdout.indexOf("Error") == -1) { + werft.log(phase, `external ip with name ${name} and ip ${ip} deleted`); + break; + } else { + werft.log(phase, `external ip with name ${name} and ip ${ip} could not be deleted, will reattempt`) + } + await sleep(5000) + } + if (attempt == 10) { + werft.log(phase, `could not delete the external ip with name ${name} and ip ${ip}`) + } +} + +function getExternalIp(name: string, region = "europe-west1") { + return exec(`gcloud compute addresses describe ${name} --region ${region}| grep 'address:' | cut -c 10-`, { silent: true }).trim(); +} diff --git a/.werft/wipe-devstaging.ts b/.werft/wipe-devstaging.ts index 80c1900566abf9..b88ab5795af7ff 100644 --- a/.werft/wipe-devstaging.ts +++ b/.werft/wipe-devstaging.ts @@ -1,13 +1,15 @@ -import { werft } from './util/shell'; +import { werft, exec } from './util/shell'; import { wipePreviewEnvironment, listAllPreviewNamespaces } from './util/kubectl'; +import * as fs from 'fs'; +import { deleteExternalIp } from './util/gcloud'; -async function wipeDevstaging() { +async function wipePreviewCluster(pathToKubeConfig: string) { const namespace_raw = process.env.NAMESPACE; const namespaces: string[] = []; if (namespace_raw === "" || !namespace_raw) { werft.log('wipe', "Going to wipe all namespaces"); - listAllPreviewNamespaces("") + listAllPreviewNamespaces(pathToKubeConfig) .map(ns => namespaces.push(ns)); } else { werft.log('wipe', `Going to wipe namespace ${namespace_raw}`); @@ -15,9 +17,39 @@ async function wipeDevstaging() { } for (const namespace of namespaces) { - await wipePreviewEnvironment("", "gitpod", namespace, { slice: 'wipe' }); + await wipePreviewEnvironment(pathToKubeConfig, "gitpod", namespace, { slice: 'wipe' }); } - werft.done('wipe'); } -wipeDevstaging() \ No newline at end of file +// if we have "/workspace/k3s-external.yaml" present that means a k3s ws cluster +// exists, therefore, delete corresponding preview deployment from that cluster too +// NOTE: Even for a non k3s ws deployment we will attempt to clean the preview. +// This saves us from writing complex logic of querying meta cluster for registered workspaces +// Since we use the same namespace to deploy in both dev and k3s cluster, this is safe +async function k3sCleanup() { + if (fs.existsSync("/workspace/k3s-external.yaml")) { + werft.log("wipe", "found /workspace/k3s-external.yaml, assuming k3s ws cluster deployment exists, will attempt to wipe it") + await wipePreviewCluster("/workspace/k3s-external.yaml") + const namespace_raw = process.env.NAMESPACE; + + // Since werft creates static external IP for ws-proxy of k3s using gcloud + // we delete it here. We retry because the ws-proxy-service which binds to this IP might not be deleted immediately + const k3sWsProxyIP = + deleteExternalIp("wipe", namespace_raw) + } else { + werft.log("wipe", `file /workspace/k3s-external.yaml does not exist, no cleanup for k3s cluster`) + } +} + +// clean up the dev cluster in gitpod-core-dev +async function devCleanup() { + await wipePreviewCluster("") +} + +// sweeper runs in the dev cluster so we need to delete the k3s cluster first and then delete self contained namespace +k3sCleanup().then(() => { + devCleanup() +}) + + +werft.done('wipe'); \ No newline at end of file diff --git a/.werft/wipe-devstaging.yaml b/.werft/wipe-devstaging.yaml index ea2b9327373351..23800adeb43710 100644 --- a/.werft/wipe-devstaging.yaml +++ b/.werft/wipe-devstaging.yaml @@ -34,6 +34,7 @@ pod: export NAMESPACE="{{ .Annotations.namespace }}" sudo chown -R gitpod:gitpod /workspace + kubectl get secret k3sdev -n werft -ojsonpath='{.data}' | jq -r .[] | base64 -d > /workspace/k3s-external.yaml npm install shelljs semver ts-node typescript @types/shelljs @types/node @types/semver npx ts-node .werft/wipe-devstaging.ts