ci: add test-integrations (#16915)

* add test-integrations workflow * add test-integrations success job * update vault integration testing versions (#16949) * change parallelism to 4 forgotestsum. use env.CONSUL_VERSION so we can see the version. * use env for repeated values * match test to circleci * fix envvar * fix envvar 2 * fix envvar 3 * fix envvar 4 * fix envvar 5 * make upgrade and compatibility tests match circleci * run go env to check environment * debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * revert debug docker Signed-off-by: Dan Bond <danbond@protonmail.com> * going back to command that worked 5 days ago for compatibility tests * Update Envoy versions to reflect changes in #16889 * cd to test dir * try running ubuntu latest * update PR with latest changes that work in enterprise * yaml still sucks * test GH fix (localhost resolution) * change for testing * test splitting and ipv6 lookup for compatibility and upgrade tests * fix indention * consul as image name * remove the on push * add gotestsum back in * removing the use of the gotestsum download action * yaml sucks today just like yesterday * fixing nomad tests * worked out the kinks on enterprise --------- Signed-off-by: Dan Bond <danbond@protonmail.com> Co-authored-by: John Eikenberry <jae@zhar.net> Co-authored-by: Dan Bond <danbond@protonmail.com> Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com> Co-authored-by: Sarah <sthompson@hashicorp.com>
hashicorp · Apr 19, 2023 · 23d2430 · 23d2430
1 parent fc20bb4
commit 23d2430
Show file tree

Hide file tree

Showing 2 changed files with 166 additions and 63 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -345,6 +345,7 @@ jobs:
             sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
           environment:
             <<: *ENVIRONMENT
+      - run: go env
       - run: *install-gotestsum
       - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
       - run:
@@ -408,6 +409,7 @@ jobs:
               sudo tar -C /usr/local -xzvf go${GO_VERSION}.linux-amd64.tar.gz
             environment:
               <<: *ENVIRONMENT
+        - run: go env
         - run: *install-gotestsum
         - run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
         - run:

diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml
@@ -21,6 +21,8 @@ env:
   GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }}
   GOTESTSUM_VERSION: "1.9.0"
   CONSUL_BINARY_UPLOAD_NAME: consul-bin
+  # strip the hashicorp/ off the front of github.repository for consul
+  CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'consul' }}
 
 jobs:
   setup:
@@ -56,10 +58,6 @@ jobs:
       matrix:
         nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16']
     steps:
-      - name: Setup gotestsum
-        uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0
-        with:
-          gotestsum_version: ${{ env.GOTESTSUM_VERSION }}
       - name: Checkout Nomad
         uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
         with:
@@ -86,13 +84,14 @@ jobs:
 
       - name: Run integration tests
         run: |
+          go install gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} && \
           gotestsum \
-              --format=short-verbose \
-              --rerun-fails \
-              --rerun-fails-report=/tmp/gotestsum-rerun-fails \
-              --packages="./command/agent/consul" \
-              --junitfile $TEST_RESULTS_DIR/results.xml -- \
-              -run TestConsul
+            --format=short-verbose \
+            --rerun-fails \
+            --rerun-fails-report=/tmp/gotestsum-rerun-fails \
+            --packages="./command/agent/consul" \
+            --junitfile $TEST_RESULTS_DIR/results.xml -- \
+            -run TestConsul
 
   vault-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }}
@@ -116,10 +115,6 @@ jobs:
         with:
           go-version-file: 'go.mod'
 
-      - uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0
-        with:
-          gotestsum_version: ${{ env.GOTESTSUM_VERSION }}
-
       - name: Install Vault
         run: |
           wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip"
@@ -129,7 +124,20 @@ jobs:
       - name: Run Connect CA Provider Tests
         run: |
           mkdir -p "${{ env.TEST_RESULTS_DIR }}"
-          make test-connect-ca-providers
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage.txt ./agent/connect/ca
+          # Run leader tests that require Vault
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-leader.txt -run Vault ./agent/consul
+          # Run agent tests that require Vault
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+            --format=short-verbose \
+            --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \
+            -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent
 
   generate-envoy-job-matrices:
     needs: [setup]
@@ -145,7 +153,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.22.7", "1.23.4", "1.24.2", "1.25.1"]
+          # envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 3 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -171,7 +179,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.22.7", "1.23.4", "1.24.2", "1.25.1"]
+        envoy-version: ["1.22.11", "1.23.8", "1.24.6", "1.25.4"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:
@@ -201,10 +209,6 @@ jobs:
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
         with:
           go-version-file: 'go.mod'
-      - name: Setup gotestsum
-        uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0
-        with:
-          gotestsum_version: ${{ env.GOTESTSUM_VERSION }}
 
       - name: fetch binary
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
@@ -233,7 +237,7 @@ jobs:
           echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
           # shellcheck disable=SC2001
           sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
-          gotestsum \
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
               --debug \
               --rerun-fails \
               --rerun-fails-report=/tmp/gotestsum-rerun-fails \
@@ -251,23 +255,45 @@ jobs:
           name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
           path: ${{ env.TEST_RESULTS_DIR }}
 
+  generate-compatibility-job-matrices:
+    needs: [setup]
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    name: Generate Compatibility Job Matrices
+    outputs:
+      compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - name: Generate Compatibility Job Matrix
+        id: set-matrix
+        env:
+          TOTAL_RUNNERS: 5
+          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
+        run: |
+          cd ./test/integration/consul-container
+          {
+            echo -n "compatibility-matrix="
+            find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
+              | grep -v util | grep -v upgrade \
+              | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
+              | jq --compact-output 'map(join(" "))'
+          } >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
   compatibility-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
     needs:
       - setup
       - dev-build
+      - generate-compatibility-job-matrices
+    strategy:
+      fail-fast: false
+      matrix:
+        test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
     steps:
-      - name: Debug Docker
-        run: docker inspect -f '{{index .Options "com.docker.network.bridge.enable_icc"}}' [network]
-
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
         with:
           go-version-file: 'go.mod'
-      - name: Setup gotestsum
-        uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0
-        with:
-          gotestsum_version: ${{ env.GOTESTSUM_VERSION }}
+      - run: go env
 
       # Build the consul:local image from the already built binary
       - name: fetch binary
@@ -279,29 +305,38 @@ jobs:
         run: chmod +x consul
 
       - name: Build consul:local image
-        run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
+        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Configure GH workaround for ipv6 loopback
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          cat /etc/hosts && echo "-----------"
+          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
+          cat /etc/hosts
       - name: Compatibility Integration Tests
         run: |
-          mkdir -p "${{ env.TEST_RESULTS_DIR }}"
-          cd ./test/integration/consul-container
-          docker run --rm consul:local consul version
-          gotestsum \
-            --raw-command \
-            --format=short-verbose \
-            --debug \
-            --rerun-fails=3 \
-            --packages="./..." \
-            -- \
-            go test \
-            -p=4 \
-            -timeout=30m \
-            -json \
-            `go list ./... | grep -v upgrade` \
-            --target-image consul \
-            --target-version local \
-            --latest-image consul \
-            --latest-version latest
-          ls -lrt
+            mkdir -p "/tmp/test-results"
+            cd ./test/integration/consul-container
+            docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+            echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
+            # shellcheck disable=SC2001
+            sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
+            go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
+              --raw-command \
+              --format=short-verbose \
+              --debug \
+              --rerun-fails=3 \
+              -- \
+              go test \
+              -p=4 \
+              -tags "${{ env.GOTAGS }}" \
+              -timeout=30m \
+              -json \
+              "${{ matrix.test-cases }}" \
+              --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+              --target-version local \
+              --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+              --latest-version latest
+            ls -lrt
         env:
           # this is needed because of incompatibility between RYUK container and circleci
           GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml
@@ -315,25 +350,81 @@ jobs:
           name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }}
           path: ${{ env.TEST_RESULTS_DIR }}
 
+  generate-upgrade-job-matrices:
+    needs: [setup]
+    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
+    name: Generate Upgrade Job Matrices
+    outputs:
+      upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }}
+    steps:
+      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version-file: 'go.mod'
+      - name: Generate Updgrade Job Matrix
+        id: set-matrix
+        env:
+          TOTAL_RUNNERS: 4
+          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
+        run: |
+          cd ./test/integration/consul-container/test/upgrade
+          {
+            echo -n "upgrade-matrix="
+            go test ./...  -list=. -json | jq -r '.Output | select (. !=null) |  select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \
+            | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \
+            | jq --compact-output 'map(join("|"))'
+          } >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
+  
   upgrade-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
+    permissions:
+      id-token: write # NOTE: this permission is explicitly required for Vault auth.
+      contents: read
     needs:
       - setup
       - dev-build
+      - generate-upgrade-job-matrices
     strategy:
+      fail-fast: false
       matrix:
         consul-version: [ "1.14", "1.15"]
+        test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
     env:
       CONSUL_VERSION: ${{ matrix.consul-version }}
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
         with:
           go-version-file: 'go.mod'
-      - name: Setup gotestsum
-        uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0
+      - run: go env
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Authenticate to Vault
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: vault-auth
+        run: vault-auth
+
+      # NOTE: ENT specific step as we store secrets in Vault.
+      - name: Fetch Secrets
+        if: ${{ endsWith(github.repository, '-enterprise') }}
+        id: secrets
+        uses: hashicorp/vault-action@v2.5.0
+        with:
+          url: ${{ steps.vault-auth.outputs.addr }}
+          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
+          token: ${{ steps.vault-auth.outputs.token }}
+          secrets: |
+              kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME;
+              kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN;
+
+      # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS.
+      - name: Login to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0
         with:
-          gotestsum_version: ${{ env.GOTESTSUM_VERSION }}
+          username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }}
+          password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}
+
 
       # Get go binary from workspace
       - name: fetch binary
@@ -344,29 +435,37 @@ jobs:
       - name: restore mode+x
         run: chmod +x consul
       - name: Build consul:local image
-        run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile .
+        run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Configure GH workaround for ipv6 loopback
+        if: ${{ !endsWith(github.repository, '-enterprise') }}
+        run: |
+          cat /etc/hosts && echo "-----------"
+          sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6-localhost ip6-loopback/g' /etc/hosts
+          cat /etc/hosts
       - name: Upgrade Integration Tests
         run: |
           mkdir -p "${{ env.TEST_RESULTS_DIR }}"
           cd ./test/integration/consul-container/test/upgrade
-          docker run --rm consul:local consul version
-          PACKAGE_NAMES=$(go list -tags "${{ env.GOTAGS }}" ./...)
-          gotestsum \
+          docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
+          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
+          # shellcheck disable=SC2001
+          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
+          go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
             --raw-command \
             --format=short-verbose \
             --debug \
             --rerun-fails=3 \
-            --packages="${PACKAGE_NAMES}" \
+            --packages="./..." \
             -- \
             go test \
             -p=4 \
             -tags "${{ env.GOTAGS }}" \
             -timeout=30m \
-            -json \
-            ./... \
-            --target-image consul \
+            -json ./... \
+            -run "${{ matrix.test-cases }}" \
+            --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
             --target-version local \
-            --latest-image consul \
+            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
             --latest-version "${{ env.CONSUL_VERSION }}"
           ls -lrt
         env:
@@ -384,12 +483,14 @@ jobs:
   test-integrations-success:
     needs: 
     - setup
-    - generate-envoy-job-matrices
     - dev-build
     - nomad-integration-test
     - vault-integration-test
+    - generate-envoy-job-matrices
     - envoy-integration-test
+    - generate-compatibility-job-matrices
     - compatibility-integration-test
+    - generate-upgrade-job-matrices
     - upgrade-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: |