enhance heuristics to delete preview-environments

Author: Wulf Thimm

.werft/platform-delete-preview-environments-cron.ts

@@ -37,48 +37,98 @@ async function deletePreviewEnvironments() {
     }
     werft.done("prep")
 
-    werft.phase("Fetching branches");
-    const branches = getAllBranches();
-    // During the transition from the old preview names to the new ones we have to check for the existence of both the old or new
-    // preview name patterns before it is safe to delete a namespace.
-    const expectedPreviewEnvironmentNamespaces = new Set(branches.flatMap(branch => [parseBranch(branch), expectedNamespaceFromBranch(branch)]));
-    werft.done("Fetching branches");
-
     werft.phase("Fetching previews");
     let previews: string[]
     try {
         previews = listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, {});
-        previews.forEach(previewNs => werft.log("Fetching previews", previewNs))
+        previews.forEach(previewNs => werft.log("Fetching previews", previewNs));
     } catch (err) {
         werft.fail("Fetching previews", err)
     }
     werft.done("Fetching previews");
 
+    werft.phase("Fetching outdated branches");
+    const branches = getAllBranches();
+    const outdatedPreviews = new Set(branches
+        .filter(branch => {
+          const lastCommit = exec(`git log ${branch} --since=$(date +%Y-%m-%d -d "5 days ago")`, { silent: true })
+          return lastCommit.length < 1
+        })
+        .map(branch => expectedNamespaceFromBranch(branch.replace('origin\/','').trim())))
+
+    const expectedPreviewEnvironmentNamespaces = new Set(branches.map(branch => expectedNamespaceFromBranch(branch)));
+
     werft.phase("deleting previews")
     try {
-        const previewsToDelete = previews.filter(ns => !expectedPreviewEnvironmentNamespaces.has(ns))
-        // Trigger namespace deletion in parallel
-        const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, CORE_DEV_KUBECONFIG_PATH, { slice: `Deleting preview ${preview}` }));
-        // But wait for all of them to finish before (or one of them to fail) before we continue
-        await Promise.all(promises)
+        const deleteDueToMissingBranch  = previews.filter(ns => !expectedPreviewEnvironmentNamespaces.has(ns))
+        const deleteDueToNoCommitActivity  = previews.filter(ns => outdatedPreviews.has(ns))
+        const deleteDueToNoDBActivity      = previews.filter(ns => checkActivity(ns))
+        const previewsToDelete             = new Set([...deleteDueToMissingBranch, ...deleteDueToNoCommitActivity, ...deleteDueToNoDBActivity])
+
+        const dryRun = true
+        if (dryRun) {
+            previewsToDelete.forEach(preview => werft.log("Would have deleted preview environment ", preview))
+        }
+        else {
+            previewsToDelete.forEach(preview => werft.log("deleting preview", preview))
+            // const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, CORE_DEV_KUBECONFIG_PATH, { slice: `Deleting preview ${preview}` }));
+            // await Promise.all(promises)
+        }
+        werft.done("deleting previews")
     } catch (err) {
         werft.fail("deleting previews", err)
     }
-    werft.done("deleting previews")
+}
+
+
+function checkActivity(previewNS) {
+
+    const statusNS = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get ns ${previewNS} -o jsonpath='{.status.phase}'`, { silent: true})
+
+    if ( statusNS == "Active") {
+
+        const emptyNS = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods -n ${previewNS} -o jsonpath='{.items.*}'`, { silent: true})
+
+        if ( emptyNS.length < 1 )
+            return;
+
+        const statusDB = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${previewNS} -o jsonpath='{.status.phase}'`, { silent: true})
+        const statusDbContainer = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get pods mysql-0 -n ${previewNS} -o jsonpath='{.status.containerStatuses.*.ready}'`, { silent: true})
+
+        if (statusDB.code == 0 && statusDB == "Running" && statusDbContainer != "false") {
+
+            const connectionToDb = `KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} kubectl get secret db-password -n ${previewNS} -o jsonpath='{.data.mysql-root-password}' | base64 -d | mysql --host=db.${previewNS}.svc.cluster.local --port=3306 --user=root --database=gitpod -s -N -p`
+
+            const latestInstanceTimeout = 24
+            const latestInstance = exec(`${connectionToDb} --execute="SELECT creationTime FROM d_b_workspace_instance WHERE creationTime > DATE_SUB(NOW(), INTERVAL '${latestInstanceTimeout}' HOUR) LIMIT 1"`, { silent: true })
+
+            const latestUserTimeout = 24
+            const latestUser= exec(`${connectionToDb} --execute="SELECT creationDate FROM d_b_user WHERE creationDate > DATE_SUB(NOW(), INTERVAL '${latestUserTimeout}' HOUR) LIMIT 1"`, { silent: true })
+
+            const lastModifiedTimeout = 24
+            const lastModified= exec(`${connectionToDb} --execute="SELECT _lastModified FROM d_b_user WHERE _lastModified > DATE_SUB(NOW(), INTERVAL '${lastModifiedTimeout}' HOUR) LIMIT 1"`, { silent: true })
+
+            const heartbeatTimeout = 24
+            const heartbeat= exec(`${connectionToDb} --execute="SELECT lastSeen FROM d_b_workspace_instance_user WHERE lastSeen > DATE_SUB(NOW(), INTERVAL '${heartbeatTimeout}' HOUR) LIMIT 1"`, { silent: true })
+
+            if ( (heartbeat.length      < 1) &&
+                 (latestInstance.length < 1) &&
+                 (latestUser.length     < 1) &&
+                 (lastModified.length   < 1) ) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
 }
 
 function getAllBranches(): string[] {
-    return exec(`git branch -r | grep -v '\\->' | sed "s,\\x1B\\[[0-9;]*[a-zA-Z],,g" | while read remote; do echo "\${remote#origin/}"; done`).stdout.trim().split('\n');
+    return exec(`git branch -r | grep -v '\\->' | sed "s,\\x1B\\[[0-9;]*[a-zA-Z],,g"`).stdout.trim().split('\n');
 }
 
 function expectedNamespaceFromBranch(branch: string): string {
     const previewName = previewNameFromBranchName(branch)
     return `staging-${previewName}`
 }
-
-function parseBranch(branch: string): string {
-    const prefix = 'staging-';
-    const parsedBranch = branch.normalize().split("/").join("-");
-
-    return prefix + parsedBranch;
-}