Prepare to remove actions hooks (#1038)

This will allow us to remove all of our `preTest` hooks in provider ci-mgmt configs - which we'd like to get rid of to avoid arbitary code injection into workflows which make them fragile and hard to refactor. See also #1037 Related to #936 ## Add standalone option for running provider integration tests Set `integrationTestProvider: true` to run `go test [...] -tags=${{ matrix.language }}` in the `provider` directory. Example of existing preTest usage: https://github.com/pulumi/pulumi-gcp/blob/92b64b51bfb05fc0d2d7b9c1cbcafe7de8a6f7f3/.ci-mgmt.yaml#L43 ### Always prepare upstream before testing This step is safe to run, as we already do before the Go build steps elsewhere. The upstream target is always there but might be a no-op. ## Add SSH setup option Set `sshPrivateKey: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}` to set up SSH for testing. Example of docker use of preTest: https://github.com/pulumi/pulumi-docker/blob/fc1d68f823c34cef72e34e060b093230e21fc636/.ci-mgmt.yaml#L35 # Migration 1. Merge this PR with intent-focused config options 2. Open PRs to the ~28 providers to replace the `preTest` hook with alternative settings - Set `integrationTestProvider: true` on all providers which run integration tests via the provider module - Use `setup-script` for arbitrary bash commands before tests - Set `sshPrivateKey: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}` for docker 4. Remove the `actions` config completely Existing usage of `actions` hooks (there is no use of `preBuild` - only `preTest`): https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22actions%3A%22&type=code
pulumi · Jul 24, 2024 · fba1faf · fba1faf
1 parent 1fc1007
commit fba1faf
Show file tree

Hide file tree

Showing 18 changed files with 101 additions and 32 deletions.
diff --git a/provider-ci/internal/pkg/templates/bridged-provider.config.yaml b/provider-ci/internal/pkg/templates/bridged-provider.config.yaml
@@ -129,6 +129,9 @@ extraTests:
 #     name: My Test
 #     ...
 
+# Run e2e tests in the provider as well as in the examples directory
+integrationTestProvider: false
+
 # Run e2e tests using the examples and test suite in the pulumi/examples repo.
 # This is unused: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22testPulumiExamples%3A%22&type=code
 testPulumiExamples: false
@@ -208,6 +211,11 @@ publish:
 # Used in 9 providers: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22docker%3A%22&type=code
 #docker: false
 
+# Setup SSH with specified private key before running tests in CI job.
+# This should be provided from a secret
+# Used by the docker provider only: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22sshPrivateKey%3A%22&type=code
+#sshPrivateKey: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}
+
 # Authenticate with GCP before running tests in CI job
 # Used in gcp and docker: https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22gcp%3A%22&type=code
 #gcp: false

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/main.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/main.yml
@@ -190,6 +190,16 @@ jobs:
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
     #{{- end }}#
+    #{{- if .Config.sshPrivateKey }}#
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.7.0
+      with:
+        ssh-private-key: #{{ .Config.sshPrivateKey }}#
+    #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Prepare upstream code
+      run: make upstream
+    #{{- end }}#
     #{{- if index .Config "setup-script" }}#
     - name: Run setup script
       run: #{{ index .Config "setup-script" }}#
@@ -204,9 +214,13 @@ jobs:
 #{{- if .Config.actions.preTest }}#
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Run provider tests
+      working-directory: provider
+      run: go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    #{{- end }}#
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/nightly-test.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/nightly-test.yml
@@ -103,6 +103,16 @@ jobs:
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
     #{{- end }}#
+    #{{- if .Config.sshPrivateKey }}#
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.7.0
+      with:
+        ssh-private-key: #{{ .Config.sshPrivateKey }}#
+    #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Prepare upstream code
+      run: make upstream
+    #{{- end }}#
     #{{- if index .Config "setup-script" }}#
     - name: Run setup script
       run: #{{ index .Config "setup-script" }}#
@@ -117,6 +127,12 @@ jobs:
 #{{- if .Config.actions.preTest }}#
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Run provider tests
+      if: matrix.testTarget == 'local'
+      working-directory: provider
+      run: go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    #{{- end }}#
     - name: Run tests
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
         matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml
@@ -131,6 +131,16 @@ jobs:
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
     #{{- end }}#
+    #{{- if .Config.sshPrivateKey }}#
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.7.0
+      with:
+        ssh-private-key: #{{ .Config.sshPrivateKey }}#
+    #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Prepare upstream code
+      run: make upstream
+    #{{- end }}#
     #{{- if index .Config "setup-script" }}#
     - name: Run setup script
       run: #{{ index .Config "setup-script" }}#
@@ -145,9 +155,13 @@ jobs:
 #{{- if .Config.actions.preTest }}#
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Run provider tests
+      working-directory: provider
+      run: go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    #{{- end }}#
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/release.yml
@@ -130,6 +130,16 @@ jobs:
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
     #{{- end }}#
+    #{{- if .Config.sshPrivateKey }}#
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.7.0
+      with:
+        ssh-private-key: #{{ .Config.sshPrivateKey }}#
+    #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Prepare upstream code
+      run: make upstream
+    #{{- end }}#
     #{{- if index .Config "setup-script" }}#
     - name: Run setup script
       run: #{{ index .Config "setup-script" }}#
@@ -144,9 +154,13 @@ jobs:
 #{{- if .Config.actions.preTest }}#
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Run provider tests
+      working-directory: provider
+      run: go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    #{{- end }}#
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/...der-ci/internal/pkg/templates/bridged-provider/.github/workflows/run-acceptance-tests.yml b/...der-ci/internal/pkg/templates/bridged-provider/.github/workflows/run-acceptance-tests.yml
@@ -173,6 +173,16 @@ jobs:
     - name: Login to Google Cloud Registry
       run: gcloud --quiet auth configure-docker
     #{{- end }}#
+    #{{- if .Config.sshPrivateKey }}#
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.7.0
+      with:
+        ssh-private-key: #{{ .Config.sshPrivateKey }}#
+    #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Prepare upstream code
+      run: make upstream
+    #{{- end }}#
     #{{- if index .Config "setup-script" }}#
     - name: Run setup script
       run: #{{ index .Config "setup-script" }}#
@@ -187,10 +197,15 @@ jobs:
 #{{- if .Config.actions.preTest }}#
 #{{ .Config.actions.preTest | toYaml | indent 4 }}#
 #{{- end }}#
+    #{{- if .Config.integrationTestProvider }}#
+    - name: Run provider tests
+      if: matrix.testTarget == 'local'
+      working-directory: provider
+      run: go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+    #{{- end }}#
     - name: Run tests
       if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run pulumi/examples tests
       if: matrix.testTarget == 'pulumiExamples'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{

diff --git a/provider-ci/test-providers/aws/.github/workflows/master.yml b/provider-ci/test-providers/aws/.github/workflows/master.yml
@@ -181,8 +181,7 @@ jobs:
     - name: Make upstream
       run: make upstream
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/aws/.github/workflows/prerelease.yml b/provider-ci/test-providers/aws/.github/workflows/prerelease.yml
@@ -124,8 +124,7 @@ jobs:
     - name: Make upstream
       run: make upstream
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/aws/.github/workflows/release.yml b/provider-ci/test-providers/aws/.github/workflows/release.yml
@@ -123,8 +123,7 @@ jobs:
     - name: Make upstream
       run: make upstream
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/aws/.github/workflows/run-acceptance-tests.yml b/provider-ci/test-providers/aws/.github/workflows/run-acceptance-tests.yml
@@ -162,8 +162,7 @@ jobs:
       run: make upstream
     - name: Run tests
       if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run pulumi/examples tests
       if: matrix.testTarget == 'pulumiExamples'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{

diff --git a/provider-ci/test-providers/cloudflare/.github/workflows/master.yml b/provider-ci/test-providers/cloudflare/.github/workflows/master.yml
@@ -166,8 +166,7 @@ jobs:
       run: |
         cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/cloudflare/.github/workflows/prerelease.yml b/provider-ci/test-providers/cloudflare/.github/workflows/prerelease.yml
@@ -111,8 +111,7 @@ jobs:
       run: |
         cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/cloudflare/.github/workflows/release.yml b/provider-ci/test-providers/cloudflare/.github/workflows/release.yml
@@ -110,8 +110,7 @@ jobs:
       run: |
         cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/cloudflare/.github/workflows/run-acceptance-tests.yml b/provider-ci/test-providers/cloudflare/.github/workflows/run-acceptance-tests.yml
@@ -153,8 +153,7 @@ jobs:
         cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run tests
       if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run pulumi/examples tests
       if: matrix.testTarget == 'pulumiExamples'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{

diff --git a/provider-ci/test-providers/docker/.github/workflows/master.yml b/provider-ci/test-providers/docker/.github/workflows/master.yml
@@ -200,8 +200,7 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/docker/.github/workflows/prerelease.yml b/provider-ci/test-providers/docker/.github/workflows/prerelease.yml
@@ -145,8 +145,7 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/docker/.github/workflows/release.yml b/provider-ci/test-providers/docker/.github/workflows/release.yml
@@ -144,8 +144,7 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}
     - name: Run tests
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     strategy:
       fail-fast: false
       matrix:

diff --git a/provider-ci/test-providers/docker/.github/workflows/run-acceptance-tests.yml b/provider-ci/test-providers/docker/.github/workflows/run-acceptance-tests.yml
@@ -187,8 +187,7 @@ jobs:
         ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY_FOR_DIGITALOCEAN }}
     - name: Run tests
       if: matrix.testTarget == 'local'
-      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{
-        matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
+      run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt
     - name: Run pulumi/examples tests
       if: matrix.testTarget == 'pulumiExamples'
       run: cd examples && go test -v -json -count=1 -cover -timeout 2h -tags=${{