Allow bridged providers to opt-in to sharding

pulumi · Dec 12, 2024 · 4f9db6c · 4f9db6c
1 parent 89a0422
commit 4f9db6c
Show file tree

Hide file tree

Showing 10 changed files with 145 additions and 35 deletions.
diff --git a/provider-ci/internal/pkg/config.go b/provider-ci/internal/pkg/config.go
@@ -254,6 +254,14 @@ type Config struct {
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22parallel%3A%22&type=code
 	Parallel int `yaml:"parallel"`
 
+	// Shards controls how many jobs integration tests are distributed across.
+	Shards int `yaml:"shards"`
+
+	// E2ETestDirectory controls the working directory when executing long-running/sharded integration tests.
+	//
+	// Defaults to "examples".
+	E2ETestDirectory string `yaml:"e2eTestDirectory"`
+
 	// Hybrid has no effect but is set by the docker provider.
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22hybrid%3A%22&type=code
 	Hybrid bool `yaml:"hybrid"`

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/test.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/test.yml
@@ -39,29 +39,50 @@ jobs:
         #{{- end }}#
         persist-credentials: false
     - name: Checkout p/examples
+      #{{- if not .Config.Shards }}#
       if: matrix.testTarget == 'pulumiExamples'
+      #{{- end }}#
       uses: #{{ .Config.ActionVersions.Checkout }}#
       with:
         repository: pulumi/examples
         path: p-examples
     - name: Setup tools
       uses: ./.github/actions/setup-tools
+      #{{- if not .Config.Shards }}#
       with:
         tools: pulumictl, pulumicli, ${{ matrix.language }}
+      #{{- end }}#
     - name: Prepare local workspace
       run: make prepare_local_workspace
     - name: Download bin
       uses: ./.github/actions/download-bin
+    #{{- if not .Config.Shards }}#
     - name: Download SDK
       uses: ./.github/actions/download-sdk
       with:
         language: ${{ matrix.language }}
     - name: Restore makefile progress
       run: make --touch provider schema build_${{ matrix.language }}
+    - name: Install dependencies
+      run: make install_${{ matrix.language}}_sdk
+    #{{- else }}#
+      #{{- range $_, $language := .Config.Languages }}#
+    - name: Download #{{ $language }}# SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: #{{ $language }}#
+    - name: Restore makefile progress
+      run: make --touch provider schema build_#{{ $language }}#
+    - name: Install dependencies
+      run: make install_#{{ $language }}#_sdk
+      #{{- end }}#
+    #{{- end }}#
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
+      #{{- if not .Config.Shards }}#
       if: matrix.language == 'python'
+      #{{- end }}#
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
@@ -102,8 +123,6 @@ jobs:
     - name: Run setup script
       run: #{{ index .Config.SetupScript }}#
     #{{- end }}#
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with:
@@ -112,6 +131,7 @@ jobs:
 #{{- if .Config.Actions.PreTest }}#
 #{{ .Config.Actions.PreTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if not .Config.Shards }}#
     #{{- if .Config.IntegrationTestProvider }}#
     - name: Run provider tests
       if: matrix.testTarget == 'local'
@@ -134,6 +154,24 @@ jobs:
         #{{- else }}#
         testTarget: [local]
         #{{- end }}#
+    #{{- else }}#
+    - name: Generate shard
+      run: make shard
+      env:
+        SHARD_TOTAL: ${{ matrix.total }}
+        SHARD_INDEX: ${{ matrix.index }}
+    - name: Run integration tests
+      run: make test_shard
+    strategy:
+       fail-fast: false
+       matrix:
+         total:
+         - #{{ .Config.Shards }}#
+         index:
+         #{{- range $i, $_ := until .Config.Shards }}#
+         - #{{ $i }}#
+         #{{- end }}#
+    #{{- end }}#
 #{{- if .Config.ExtraTests }}#
 #{{ .Config.ExtraTests | toYaml | indent 2 }}#
 #{{ end }}#
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/Makefile b/provider-ci/internal/pkg/templates/bridged-provider/Makefile
@@ -258,7 +258,7 @@ bin/$(PROVIDER): .make/schema
 
 test: export PATH := $(WORKING_DIR)/bin:$(PATH)
 test:
-	cd examples && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
+	cd #{{ .Config.E2ETestDirectory }}# && go test -v -tags=all -parallel $(TESTPARALLELISM) -timeout 2h
 .PHONY: test
 
 test_provider:
@@ -379,6 +379,15 @@ provider_dist-darwin-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar
 provider_dist-windows-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz
 provider_dist: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64
 .PHONY: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 provider_dist
-
+#{{ if .Config.Shards -}}#
+# shard computes tests to run and modifies the CI runner's environment.
+shard:
+	@(cd #{{ .Config.E2ETestDirectory }}# && go run github.com/blampe/shard@latest --total $(SHARD_TOTAL) --index $(SHARD_INDEX) --output env) >> "$(GITHUB_ENV)"
+
+# test_shard runs the tests specified by a regex contained in $SHARD_TESTS for paths $SHARD_PATHS.
+test_shard:
+	cd #{{ .Config.E2ETestDirectory }}# && \
+		go test -tags=all -v -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 3h -parallel ${TESTPARALLELISM} -run "$(SHARD_TESTS)" $(SHARD_PATHS)
+#{{- end }}#
 # Permit providers to extend the Makefile with provider-specific Make includes.
 include $(wildcard .mk/*.mk)
diff --git a/provider-ci/internal/pkg/templates/defaults.config.yaml b/provider-ci/internal/pkg/templates/defaults.config.yaml
@@ -124,6 +124,8 @@ env:
   # Bridge-related - could be set in makefile instead?
   TF_APPEND_USER_AGENT: pulumi
 
+# DEPRECATED
+#
 # actions can contain preBuild and preTest additional steps to be spliced into workflows.
 # The use of these hooks vary - quite a few just build upstream and run provider tests.
 # Usage: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22actions%3A%22&type=code
@@ -134,6 +136,8 @@ actions:
   # preTest:
   #  - Your action here
 
+# DEPRECATED
+#
 # Additional tests run as part of `run-acceptance-tests.yml`, `master.yml`, `main.yml`,
 # `prerelease.yml` and `release.yml`.
 # Only used for aws: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22extraTests%3A%22&type=code
@@ -145,10 +149,18 @@ extraTests:
 # Run e2e tests in the provider as well as in the examples directory
 integrationTestProvider: false
 
+# DEPRECATED
+#
 # Run e2e tests using the examples and test suite in the pulumi/examples repo.
 # This is unused: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22testPulumiExamples%3A%22&type=code
 testPulumiExamples: false
 
+# How many shared to execute integration tests with. If omitted, shard behavior defaults to language-based sharding.
+shards: 0
+
+# The working directory for running integration tests.
+e2eTestDirectory: examples
+
 # runner defines the runs-on property for various stages of the build
 # These are not overridden by any providers: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22runner%3A%22&type=code
 runner:

diff --git a/provider-ci/test-providers/acme/.github/workflows/test.yml b/provider-ci/test-providers/acme/.github/workflows/test.yml
@@ -61,15 +61,15 @@ jobs:
         language: ${{ matrix.language }}
     - name: Restore makefile progress
       run: make --touch provider schema build_${{ matrix.language }}
+    - name: Install dependencies
+      run: make install_${{ matrix.language}}_sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
       if: matrix.language == 'python'
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with:

diff --git a/provider-ci/test-providers/aws/.ci-mgmt.yaml b/provider-ci/test-providers/aws/.ci-mgmt.yaml
@@ -2,6 +2,7 @@ provider: aws
 lint: false
 major-version: 6
 parallel: 1
+shards: 10
 timeout: 150
 generate-nightly-test-workflow: true
 providerVersion: github.com/hashicorp/terraform-provider-aws/version.ProviderVersion

diff --git a/provider-ci/test-providers/aws/.github/workflows/test.yml b/provider-ci/test-providers/aws/.github/workflows/test.yml
@@ -53,34 +53,62 @@ jobs:
         submodules: true
         persist-credentials: false
     - name: Checkout p/examples
-      if: matrix.testTarget == 'pulumiExamples'
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         repository: pulumi/examples
         path: p-examples
     - name: Setup tools
       uses: ./.github/actions/setup-tools
-      with:
-        tools: pulumictl, pulumicli, ${{ matrix.language }}
     - name: Prepare local workspace
       run: make prepare_local_workspace
     - name: Download bin
       uses: ./.github/actions/download-bin
-    - name: Download SDK
+    - name: Download nodejs SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: nodejs
+    - name: Restore makefile progress
+      run: make --touch provider schema build_nodejs
+    - name: Install dependencies
+      run: make install_nodejs_sdk
+    - name: Download python SDK
       uses: ./.github/actions/download-sdk
       with:
-        language: ${{ matrix.language }}
+        language: python
     - name: Restore makefile progress
-      run: make --touch provider schema build_${{ matrix.language }}
+      run: make --touch provider schema build_python
+    - name: Install dependencies
+      run: make install_python_sdk
+    - name: Download dotnet SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: dotnet
+    - name: Restore makefile progress
+      run: make --touch provider schema build_dotnet
+    - name: Install dependencies
+      run: make install_dotnet_sdk
+    - name: Download go SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: go
+    - name: Restore makefile progress
+      run: make --touch provider schema build_go
+    - name: Install dependencies
+      run: make install_go_sdk
+    - name: Download java SDK
+      uses: ./.github/actions/download-sdk
+      with:
+        language: java
+    - name: Restore makefile progress
+      run: make --touch provider schema build_java
+    - name: Install dependencies
+      run: make install_java_sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
-      if: matrix.language == 'python'
       run: |-
         pip3 install virtualenv==20.0.23
         pip3 install pipenv
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with:
@@ -97,22 +125,29 @@ jobs:
         role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
     - name: Make upstream
       run: make upstream
-    - name: Run tests
-      if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 .
-    - name: Run pulumi/examples tests
-      if: matrix.testTarget == 'pulumiExamples'
-      run: cd examples && go test -v -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 4 .
+    - name: Generate shard
+      run: make shard
+      env:
+        SHARD_TOTAL: ${{ matrix.total }}
+        SHARD_INDEX: ${{ matrix.index }}
+    - name: Run integration tests
+      run: make test_shard
     strategy:
-      fail-fast: false
-      matrix:
-        language:
-        - nodejs
-        - python
-        - dotnet
-        - go
-        - java
-        testTarget: [local]
+       fail-fast: false
+       matrix:
+         total:
+         - 10
+         index:
+         - 0
+         - 1
+         - 2
+         - 3
+         - 4
+         - 5
+         - 6
+         - 7
+         - 8
+         - 9
   go_test_shim:
       name: Run test of provider shim
       runs-on: ubuntu-latest

diff --git a/provider-ci/test-providers/aws/Makefile b/provider-ci/test-providers/aws/Makefile
@@ -351,6 +351,13 @@ provider_dist-darwin-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar
 provider_dist-windows-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz
 provider_dist: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64
 .PHONY: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 provider_dist
-
+# shard computes tests to run and modifies the CI runner's environment.
+shard:
+	@(cd examples && go run github.com/blampe/shard@latest --total $(SHARD_TOTAL) --index $(SHARD_INDEX) --output env) >> "$(GITHUB_ENV)"
+
+# test_shard runs the tests specified by a regex contained in $SHARD_TESTS for paths $SHARD_PATHS.
+test_shard:
+	cd examples && \
+		go test -tags=all -v -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 3h -parallel ${TESTPARALLELISM} -run "$(SHARD_TESTS)" $(SHARD_PATHS)
 # Permit providers to extend the Makefile with provider-specific Make includes.
 include $(wildcard .mk/*.mk)
diff --git a/provider-ci/test-providers/cloudflare/.github/workflows/test.yml b/provider-ci/test-providers/cloudflare/.github/workflows/test.yml
@@ -63,6 +63,8 @@ jobs:
         language: ${{ matrix.language }}
     - name: Restore makefile progress
       run: make --touch provider schema build_${{ matrix.language }}
+    - name: Install dependencies
+      run: make install_${{ matrix.language}}_sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
@@ -72,8 +74,6 @@ jobs:
         pip3 install pipenv
     - name: Run setup script
       run: example/script.sh
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with:

diff --git a/provider-ci/test-providers/docker/.github/workflows/test.yml b/provider-ci/test-providers/docker/.github/workflows/test.yml
@@ -76,6 +76,8 @@ jobs:
         language: ${{ matrix.language }}
     - name: Restore makefile progress
       run: make --touch provider schema build_${{ matrix.language }}
+    - name: Install dependencies
+      run: make install_${{ matrix.language}}_sdk
     - name: Update path
       run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
     - name: Install Python deps
@@ -106,8 +108,6 @@ jobs:
         install_components: gke-gcloud-auth-plugin
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
-    - name: Install dependencies
-      run: make install_${{ matrix.language}}_sdk
     - name: Install gotestfmt
       uses: GoTestTools/gotestfmt-action@v2
       with: