ci: integration tests pipeline improvements

.github/workflows/nightly-build.yml

@@ -5,7 +5,7 @@ on:
     - cron: '0 10 * * *' # Run every day at 10AM UTC
 
 jobs:
-  run-tests:
+  run-linting-and-unit-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -18,7 +18,7 @@ jobs:
         with:
           go-version: 1.21.x
 
-      - name: Run linting tests
+      - name: Run linting
         id: update
         run: |
           make prepare
@@ -30,9 +30,62 @@ jobs:
         run: |
           make coverage-ci
 
+      # TODO Display test report
+
+      - name: Notify Slack on Failure
+        uses: slackapi/slack-github-action@v1.25.0
+        if: failure()
+        with:
+          payload: |
+            {
+              "attachments": [
+                {
+                  "color": "#E92020",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "@oncall-growth-eng! There has been a failure that needs your attention. :rotating_light:\n*GitHub Workflow Failure*\ngo-sdk/nightly-build\n*Workflow Run*\n https://github.com/lacework/go-sdk/actions/runs/${{ github.run_id }}"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_GROWTH_ENG_ALERTS }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+  run-integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # To run integration tests with different tags in parallel
+        index: [0, 1, 2, 3, 4]
+    steps:
+      # To wait for the existing nightly-build run to complete to avoid running same integration tests at the same time
+      - name: Turnstyle
+        uses: softprops/turnstyle@v1
+        with:
+          same-branch-only: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.x
+
       - name: Build cross-platform binaries
         run: |
-          make build-cli-cross-platform
+          make prepare
+          make build
 
       - name: Run integration tests
         env:
@@ -41,11 +94,9 @@ jobs:
           CI_API_KEY: ${{ secrets.CI_API_KEY }}
           CI_API_SECRET: ${{ secrets.CI_API_SECRET }}
           LW_INT_TEST_AWS_ACC: ${{ secrets.LW_INT_TEST_AWS_ACC }}
-        # need vim for integrations test of inline editing LQL queries
+          LW_CLI_BIN: lacework
         run: |
-          sudo apt-get update
-          sudo apt-get install -y vim
-          make integration-only
+          make integration-only-subset index=${{ matrix.index }}
 
       - name: Notify Slack on Failure
         uses: slackapi/slack-github-action@v1.25.0

.github/workflows/test-build.yml

@@ -8,17 +8,9 @@ on:
       - main
 
 jobs:
-  run-tests:
+  run-linting-and-unit-tests:
     runs-on: ubuntu-latest
     steps:
-      # To wait for the existing test-build run to complete
-      - name: Turnstyle
-        uses: softprops/turnstyle@v1
-        with:
-          same-branch-only: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.TOKEN }}
-
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -29,7 +21,7 @@ jobs:
         with:
           go-version: 1.21.x
 
-      - name: Run linting tests
+      - name: Run linting
         id: update
         run: |
           make prepare
@@ -41,9 +33,62 @@ jobs:
         run: |
           make coverage-ci
 
+      # TODO Display test report
+
+      - name: Notify Slack on Failure
+        uses: slackapi/slack-github-action@v1.25.0
+        if: failure()
+        with:
+          payload: |
+            {
+              "attachments": [
+                {
+                  "color": "#E92020",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "@oncall-growth-eng! There has been a failure that needs your attention. :rotating_light:\n*GitHub Workflow Failure*\ngo-sdk/test-build\n*Workflow Run*\n https://github.com/lacework/go-sdk/actions/runs/${{ github.run_id }}"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_GROWTH_ENG_ALERTS }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+  run-integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # To run integration tests with different tags in parallel
+        index: [0, 1, 2, 3, 4]
+    steps:
+      # To wait for the existing test-build run to complete to avoid running same integration tests at the same time
+      - name: Turnstyle
+        uses: softprops/turnstyle@v1
+        with:
+          same-branch-only: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOKEN }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.x
+
       - name: Build cross-platform binaries
         run: |
-          make build-cli-cross-platform
+          make prepare
+          make build
 
       - name: Run integration tests
         env:
@@ -52,12 +97,9 @@ jobs:
           CI_API_KEY: ${{ secrets.CI_API_KEY }}
           CI_API_SECRET: ${{ secrets.CI_API_SECRET }}
           LW_INT_TEST_AWS_ACC: ${{ secrets.LW_INT_TEST_AWS_ACC }}
+          LW_CLI_BIN: lacework
         run: |
-          sudo apt-get update
-          sudo apt-get install -y vim
-          make integration-only
-
-      # TODO Display test report
+          make integration-only-subset index=${{ matrix.index }}
 
       - name: Notify Slack on Failure
         uses: slackapi/slack-github-action@v1.25.0
@@ -85,7 +127,7 @@ jobs:
           SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   trigger-release:
-    needs: run-tests
+    needs: [run-linting-and-unit-tests, run-integration-tests]
     if: github.ref_name == 'main'
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -27,6 +27,28 @@ GOFLAGS=-mod=vendor
 CGO_ENABLED?=0
 export GOFLAGS GO_LDFLAGS CGO_ENABLED
 
+INTEGRATION_TEST_TAGS=account \
+	agent_token \
+	alert \
+	alert_rule \
+	alert_channel \
+	alert_profile \
+	agent \
+	configure \
+	cloud_account \
+	container_registry \
+	query \
+	policy \
+	help \
+	version \
+	compliance \
+	team_member \
+	vulnerability \
+	report_definitions \
+	component \
+	resource_groups \
+	generation
+
 .PHONY: help
 help:
 	@echo "-------------------------------------------------------------------"
@@ -40,7 +62,9 @@ prepare: git-env install-tools go-vendor ## Initialize the go environment
 
 .PHONY: test
 test: prepare ## Run all go-sdk tests
-	gotestsum -f testname -- -v -cover -run=$(regex) -coverprofile=$(COVERAGEOUT) $(shell go list ./... | grep -v integration)
+	$(eval PACKAGES := $(shell go list ./... | grep -v integration))
+	gotestsum -f testname --rerun-fails=3 --packages="$(PACKAGES)" \
+		-- -v -cover -run=$(regex) -coverprofile=$(COVERAGEOUT) $(PACKAGES)
 
 .PHONY: integration
 integration: build-cli-cross-platform integration-only ## Build and run integration tests
@@ -58,28 +82,20 @@ integration-generation-only: ## Run integration tests
 
 .PHONY: integration-only
 integration-only: install-tools ## Run integration tests
-	PATH="$(PWD)/bin:${PATH}" gotestsum -- -v github.com/lacework/go-sdk/integration -timeout 30m -tags="\
-		account \
-		agent_token \
-		alert \
-		alert_rule \
-		alert_channel \
-		alert_profile \
-		agent \
-		configure \
-		cloud_account \
-		container_registry \
-		query \
-		policy \
-		help \
-		version \
-		generation \
-		compliance \
-		team_member \
-		vulnerability \
-		report_definitions \
-		component \
-		resource_groups" -run=$(regex)
+	PATH="$(PWD)/bin:${PATH}" gotestsum -f testname  --rerun-fails=3 --packages="github.com/lacework/go-sdk/integration" \
+		-- -v github.com/lacework/go-sdk/integration -timeout 30m -tags="$(INTEGRATION_TEST_TAGS)" -run=$(regex)
+
+.PHONY: integration-only-subset
+integration-only-subset: install-tools ## Run a subset of integration tests
+	$(eval START := $(shell echo 1+$(index)*5 | bc))
+	$(eval END := $(shell echo 5+$(index)*5 | bc))
+	$(eval LENGTH := ${words $(INTEGRATION_TEST_TAGS)})
+	if [ ${START} -le ${LENGTH} ]; then \
+		PATH="$(PWD)/bin:${PATH}" gotestsum -f testname  --rerun-fails=3 --packages="github.com/lacework/go-sdk/integration" \
+			-- -v github.com/lacework/go-sdk/integration -timeout 30m \
+			-tags="${wordlist $(START), $(END), $(INTEGRATION_TEST_TAGS)}" -run=$(regex) \
+		exit 1; \
+	fi
 
 .PHONY: integration-lql
 integration-lql: build-cli-cross-platform integration-lql-only ## Build and run lql integration tests

cli/cmd/component_dev.go

@@ -643,6 +643,10 @@ func cdkGoRunVerify(componentName string) error {
 }
 
 func laceworkCLIBinary() string {
+	if bin := os.Getenv("LW_CLI_BIN"); bin != "" {
+		return bin
+	}
+
 	if os.Getenv("LW_CLI_INTEGRATION_MODE") != "" {
 		return fmt.Sprintf(
 			"lacework-cli-%s-%s",

integration/alert_rules_test.go

@@ -1,3 +1,5 @@
+//go:build alert
+
 // Author:: Darren Murray (<darren.murray@lacework.net>)
 // Copyright:: Copyright 2021, Lacework Inc.
 // License:: Apache License, Version 2.0

integration/lql_create_test.go

@@ -38,8 +38,7 @@ func TestQueryCreateHelp(t *testing.T) {
 
 func TestQueryCreateEditor(t *testing.T) {
 	out, err, exitcode := LaceworkCLIWithTOMLConfig("query", "create")
-	assert.Contains(t, out.String(), "Type a query to create")
-	assert.Contains(t, out.String(), "[Enter to launch editor]")
+	assert.Contains(t, out.String(), "Choose query language")
 	assert.Contains(t, err.String(), "ERROR unable to create query:")
 	assert.Equal(t, 1, exitcode, "EXITCODE is not the expected one")
 }

integration/lql_create_unix_test.go

@@ -38,6 +38,9 @@ func TestQueryCreateFromEditor(t *testing.T) {
 
 	_ = runFakeTerminalTestFromDir(t, dir,
 		func(c *expect.Console) {
+			expectString(t, c, "Choose query language")
+			c.SendLine("")
+			time.Sleep(time.Millisecond)
 			expectString(t, c, "Type a query to create")
 			c.SendLine("")
 			time.Sleep(time.Millisecond)