Send slack notification in IDE integration tests

Author: jeanp413

.werft/build.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import { SpanStatusCode } from '@opentelemetry/api';
 import { Werft } from './util/werft';
-import { reportBuildFailureInSlack } from './util/slack';
+import { reportToSlack } from './util/slack';
 import * as Tracing from './observability/tracing'
 import * as VM from './vm/vm'
 import { buildAndPublish } from './jobs/build/build-and-publish';
@@ -31,14 +31,16 @@ Tracing.initialize()
         werft.endAllSpans()
 
         if (context.Repository.ref === "refs/heads/main") {
-            reportBuildFailureInSlack(context, err, () => process.exit(1));
+            reportErrorToSlack(context, err).then(() => {
+                VM.stopKubectlPortForwards();
+                process.exit(1);
+            });
         } else {
             console.log('Error', err)
             // Explicitly not using process.exit as we need to flush tracing, see tracing.js
             process.exitCode = 1
+            VM.stopKubectlPortForwards();
         }
-
-        VM.stopKubectlPortForwards()
     })
 
 async function run(context: any) {
@@ -58,3 +60,14 @@ async function run(context: any) {
     await deployToPreviewEnvironment(werft, config)
     await triggerIntegrationTests(werft, config, context.Owner)
 }
+
+async function reportErrorToSlack(context: any, error: any) {
+    const repo = context.Repository.host + "/" + context.Repository.owner + "/" + context.Repository.repo;
+    const message = ":X: *build failure*\n_Repo:_ `" + repo + "`\n_Build:_ `" + context.Name + "`";
+    const werftJobUrl = "https://werft.gitpod-dev.com/job/" + context.Name;
+    try {
+        await reportToSlack(error, message, werftJobUrl);
+    } catch (err) {
+        console.log("Slack webhook notification failed", err);
+    }
+}

.werft/integration-tests-startup.yaml

@@ -84,7 +84,7 @@ pod:
 
       if ! werft job logs ${TEST_BUILD_ID} | werft log slice "test";
       then
-          echo "ingetration test failed" | werft log slice "test"
+          echo "integration test failed" | werft log slice "test"
           exit 1
       fi
-      echo "ingetration test success" | werft log slice "test"
+      echo "integration test success" | werft log slice "test"

.werft/notifySlack.ts

@@ -0,0 +1,21 @@
+import { reportToSlack } from './util/slack';
+
+async function main(args: string[]) {
+    if (args.length < 3) {
+        return;
+    }
+
+    const error = args[0];
+    const message = args[1];
+    const werftJobUrl = args[2];
+    try {
+        await reportToSlack(error, message, werftJobUrl);
+    } catch (err) {
+        console.log("Slack webhook notification failed", err);
+    }
+}
+
+if (require.main === module) {
+    main(process.argv.slice(2))
+        .then(() => process.exit(0))
+}

.werft/run-integration-tests-ide.yaml

@@ -72,14 +72,19 @@ pod:
         secretKeyRef:
           name: github-roboquat-automatic-changelog
           key: token
+    - name: SLACK_NOTIFICATION_PATH
+      valueFrom:
+        secretKeyRef:
+          name: slack-webhook-urls
+          key: ide_jobs
     command:
     - /bin/bash
     - -c
     - |
       sleep 1
       set -Eeuo pipefail
 
-      printf '{{ toJson .Annotations }}' > context.json
+      printf '{{ toJson . }}' > context.json
 
       echo "[prep] receiving config..."
       export GOOGLE_APPLICATION_CREDENTIALS="/config/gcloud/legacy_credentials/cd-gitpod-deployer@gitpod-core-dev.iam.gserviceaccount.com/adc.json"
@@ -105,6 +110,16 @@ pod:
       RC=${PIPESTATUS[0]}
       if [ $RC -eq 1 ]; then
         echo "[int-tests|FAIL]"
+
+        context_name = $(jq -r '.Name' context.json)
+        context_host = $(jq -r '.Repository.host' context.json)
+        context_owner = $(jq -r '.Repository.owner' context.json)
+        context_repo = $(jq -r '.Repository.repo' context.json)
+        repo = "${context_host}/${context_owner}/${context_repo}"
+        message = ":X: *IDE integration test failure*\n_Repo:_ ${context_repo}\n_Build:_ ${context_name}";
+        werftJobUrl = "https://werft.gitpod-dev.com/job/${context_name}"
+        errorMsg = "Some IDE integration test failed, please check the werf job logs and try to fix them"
+        npx ts-node .werft/notifySlack.ts "$errorMsg" "$message" "$werftJobUrl"
       else
         echo "[int-tests|DONE]"
       fi

.werft/util/slack.ts

@@ -1,14 +1,13 @@
 import * as https from 'https';
 
-export function reportBuildFailureInSlack(context, err, onErr) {
-    const repo = context.Repository.host + "/" + context.Repository.owner + "/" + context.Repository.repo;
+export function reportToSlack(error:string, message:string, werfJobUrl:string): Promise<void> {
     const data = JSON.stringify({
         "blocks": [
             {
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": ":X: *build failure*\n_Repo:_ `" + repo + "`\n_Build:_ `" + context.Name + "`"
+                    "text": message
                 },
                 "accessory": {
                     "type": "button",
@@ -18,30 +17,39 @@ export function reportBuildFailureInSlack(context, err, onErr) {
                         "emoji": true
                     },
                     "value": "click_me_123",
-                    "url": "https://werft.gitpod-dev.com/job/" + context.Name,
+                    "url": werfJobUrl,
                     "action_id": "button-action"
                 }
             },
             {
                 "type": "section",
                 "text": {
                     "type": "mrkdwn",
-                    "text": "```\n" + err + "\n```"
+                    "text": "```\n" + error + "\n```"
                 }
             }
         ]
     });
-    const req = https.request({
-        hostname: "hooks.slack.com",
-        port: 443,
-        path: process.env.SLACK_NOTIFICATION_PATH.trim(),
-        method: "POST",
-        headers: {
-            'Content-Type': 'application/json',
-            'Content-Length': data.length,
+
+    return new Promise((resolve, reject) => {
+        const options: https.RequestOptions = {
+            hostname: "hooks.slack.com",
+            port: 443,
+            path: process.env.SLACK_NOTIFICATION_PATH.trim(),
+            method: "POST",
+            headers: {
+                'Content-Type': 'application/json',
+                'Content-Length': data.length,
+            }
         }
-    }, onErr);
-    req.on('error', onErr);
-    req.write(data);
-    req.end();
+        const req = https.request(options, (res) => {
+            if (res.statusCode < 200 || res.statusCode >= 300) {
+                return reject(new Error('statusCode=' + res.statusCode));
+            }
+            res.on('end', () => resolve());
+        });
+        req.on('error', (err) => reject(err));
+        req.write(data);
+        req.end();
+    });
 }