Production smoke tests (#1322)

.github/workflows/prod-e2e-test.yml

@@ -0,0 +1,83 @@
+name: Botkube Cloud Prod E2E test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */3 * * *" # Every 3 hours
+
+env:
+  HELM_VERSION: v3.9.0
+  K3D_VERSION: v5.4.6
+  IMAGE_REGISTRY: "ghcr.io"
+  IMAGE_REPOSITORY: "kubeshop/botkube"
+  CFG_EXPORTER_IMAGE_REPOSITORY: "kubeshop/botkube-config-exporter"
+  IMAGE_TAG: v9.99.9-dev # TODO: Use commit hash tag to make the predictable builds for each commit on branch
+
+jobs:
+  cloud-slack-prod-e2e:
+    name: Botkube Cloud Slack Prod E2E
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    concurrency:
+      group: cloud-slack-prod-e2e
+      cancel-in-progress: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Download k3d
+        run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash"
+      - name: Create k3d cluster
+        run: "k3d cluster create cloud-slack-prod-e2e-cluster --wait --timeout=5m"
+
+      - name: Run e2e tests
+        env:
+          SLACK_WORKSPACE_NAME: ${{ secrets.E2E_DEV_SLACK_WORKSPACE_NAME }}
+          SLACK_EMAIL: ${{ secrets.E2E_DEV_SLACK_EMAIL }}
+          SLACK_PASSWORD: ${{ secrets.E2E_DEV_SLACK_USER_PASSWORD }}
+          SLACK_BOT_DISPLAY_NAME: Botkube
+          SLACK_TESTER_TESTER_BOT_TOKEN: ${{ secrets.E2E_DEV_SLACK_TESTER_BOT_TOKEN }}
+          SLACK_TESTER_BOT_NAME: botkube3
+          SLACK_TESTER_MESSAGE_WAIT_TIMEOUT: 90s
+          BOTKUBE_CLOUD_API_BASE_URL: https://api.botkube.io
+          BOTKUBE_CLOUD_EMAIL: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_EMAIL }}
+          BOTKUBE_CLOUD_PASSWORD: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_PASSWORD }}
+          BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID }}
+          BOTKUBE_CLOUD_FREE_ORGANIZATION_ID: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_FREE_ORGANIZATION_ID }}
+          BOTKUBE_CLOUD_PLUGIN_REPO_URL: https://storage.googleapis.com/botkube-plugins-latest/plugins-index.yaml
+          SCREENSHOTS_DIR: ${{ runner.temp }}/screenshots
+          DEBUG_MODE: true
+        run: |
+          KUBECONFIG=$(k3d kubeconfig write cloud-slack-prod-e2e-cluster) make test-cloud-slack-dev-e2e
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshots_dump_${{github.sha}}
+          path: ${{ runner.temp }}/screenshots
+          retention-days: 5
+
+      - name: Dump cluster state
+        if: ${{ failure() }}
+        uses: ./.github/actions/dump-cluster
+
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        if: ${{ failure() }}
+        env:
+          SLACK_USERNAME: Botkube Cloud CI
+          SLACK_COLOR: 'red'
+          SLACK_TITLE: 'Message'
+          SLACK_CHANNEL: 'botkube-cloud-ci-alerts'
+          SLACK_MESSAGE: 'Cloud Slack Prod E2E tests failed :scream:'
+          SLACK_ICON_EMOJI: ':this-is-fine-fire:'
+          SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_PROD_ALERTS_WEBHOOK }}

test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go

@@ -81,6 +81,7 @@ type BotkubeCloudConfig struct {
 
 	TeamOrganizationID string
 	FreeOrganizationID string
+	PluginRepoURL      string `envconfig:"default=https://storage.googleapis.com/botkube-plugins-latest/plugins-dev-index.yaml"`
 }
 
 func TestCloudSlackE2E(t *testing.T) {
@@ -197,6 +198,15 @@ func TestCloudSlackE2E(t *testing.T) {
 		slackPage.MustElementR(".c-scrollbar__child .c-virtual_list__scroll_container .p-channel_sidebar__static_list__item .p-channel_sidebar__name span", fmt.Sprintf("^%s$", cfg.Slack.BotDisplayName)).MustParent().MustParent().MustParent().MustClick()
 		screenshotIfShould(t, cfg, slackPage)
 
+		// it shows a popup for the new slack UI
+		elem, _ = shortTimeoutPage.ElementR("button.c-button", "I’ll Explore on My Own")
+		if elem != nil {
+			t.Log("Closing the 'New Slack UI' modal...")
+			elem.MustClick()
+			time.Sleep(cfg.DefaultWaitTime) // to ensure the additional screenshot we do below shows closed modal
+			screenshotIfShould(t, cfg, slackPage)
+		}
+
 		t.Log("Clicking 'Connect' button...")
 		slackPage.MustElement(".p-actions_block__action button.c-button") // workaround for `MustElements` not having built-in retry
 		screenshotIfShould(t, cfg, slackPage)
@@ -328,11 +338,12 @@ func TestCloudSlackE2E(t *testing.T) {
 		})
 
 		params := helmx.InstallChartParams{
-			RepoURL:   "https://storage.googleapis.com/botkube-latest-main-charts",
-			RepoName:  "botkube",
-			Name:      "botkube",
-			Namespace: "botkube",
-			Command:   *deployment.HelmCommand,
+			RepoURL:       "https://storage.googleapis.com/botkube-latest-main-charts",
+			RepoName:      "botkube",
+			Name:          "botkube",
+			Namespace:     "botkube",
+			Command:       *deployment.HelmCommand,
+			PluginRepoURL: cfg.BotkubeCloud.PluginRepoURL,
 		}
 		helmInstallCallback := helmx.InstallChart(t, params)
 		t.Cleanup(func() {
@@ -373,7 +384,7 @@ func TestCloudSlackE2E(t *testing.T) {
 			t.Log("Testing ping for not existing deployment")
 			command = "ping"
 			deployName := "non-existing-deployment"
-			expectedMessage = "*Instance not found* The cluster non-existing-deployment does not exist."
+			expectedMessage = fmt.Sprintf("*Instance not found* The cluster %q does not exist.", deployName)
 			tester.PostMessageToBot(t, channel.ID(), fmt.Sprintf("%s --cluster-name %s", command, deployName))
 			err = tester.WaitForLastMessageContains(tester.BotUserID(), channel.ID(), expectedMessage)
 			require.NoError(t, err)
@@ -453,8 +464,15 @@ func TestCloudSlackE2E(t *testing.T) {
 
 			t.Log("Waiting for watch begin message...")
 			expectedWatchBeginMsg := fmt.Sprintf("My watch begins for cluster '%s'! :crossed_swords:", deployment.Name)
-			err = tester.WaitForLastMessageEqual(tester.BotUserID(), channel.ID(), expectedWatchBeginMsg)
-			require.NoError(t, err)
+			recentMessages := 2 // take into the account the optional "upgrade checker message"
+			err = tester.WaitForMessagePosted(tester.BotUserID(), channel.ID(), recentMessages, func(msg string) (bool, int, string) {
+				if !strings.EqualFold(expectedWatchBeginMsg, msg) {
+					count := diff.CountMatchBlock(expectedWatchBeginMsg, msg)
+					msgDiff := diff.Diff(expectedWatchBeginMsg, msg)
+					return false, count, msgDiff
+				}
+				return true, 0, ""
+			})
 
 			t.Log("Verifying disabled notification on Cloud...")
 			deploy := gqlCli.MustGetDeployment(t, graphql.ID(deployment.ID))
@@ -547,13 +565,6 @@ func TestCloudSlackE2E(t *testing.T) {
 			gotSrcEvents := SourceEmittedEventsFromAuditResponse(auditPage.Audits)
 			require.ElementsMatch(t, wantSrcEvents, gotSrcEvents)
 		})
-
-		t.Run("Try to create deployment with Cloud Slack in free tier", func(t *testing.T) {
-			gqlCli := cloud_graphql.NewClientForAuthAndOrg(gqlEndpoint, cfg.BotkubeCloud.FreeOrganizationID, authHeaderValue)
-			_, err := gqlCli.CreateBasicDeploymentWithCloudSlack(t, "it won't be created anyway", slackWorkspace.TeamID, channel.Name())
-			require.Error(t, err)
-			require.Contains(t, err.Error(), "you cannot use Cloud Slack within your plan")
-		})
 	})
 }
 

test/helmx/helm_helpers.go

@@ -2,6 +2,7 @@ package helmx
 
 import (
 	"encoding/json"
+	"fmt"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -12,11 +13,12 @@ import (
 
 // InstallChartParams are parameters for InstallChart.
 type InstallChartParams struct {
-	RepoName  string
-	RepoURL   string
-	Name      string
-	Namespace string
-	Command   string
+	RepoName      string
+	RepoURL       string
+	Name          string
+	Namespace     string
+	Command       string
+	PluginRepoURL string
 }
 
 type versionsResponse struct {
@@ -29,7 +31,15 @@ func (p *InstallChartParams) ToOptions(version string) []string {
 	cmd = strings.Replace(cmd, "\\", " ", -1)
 	versionRegex := regexp.MustCompile(`--version (\S+)`)
 	cmd = versionRegex.ReplaceAllString(cmd, "--version "+version)
-	return strings.Fields(cmd)[1:]
+	cmdParts := strings.Fields(cmd)[1:]
+	extraEnvs := []string{
+		"--set",
+		fmt.Sprintf("extraEnv[0].name=%s", "BOTKUBE_PLUGINS_REPOSITORIES_BOTKUBE_URL"),
+		"--set-string",
+		fmt.Sprintf("extraEnv[0].value=%s", p.PluginRepoURL),
+	}
+	cmdParts = append(cmdParts, extraEnvs...)
+	return cmdParts
 }
 
 // InstallChart installs helm chart.