From 5235cd676fb6b5db85cac643043302bb447bf17b Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 11 Dec 2024 15:35:15 -0800 Subject: [PATCH 1/5] Test sharding --- .github/actions/setup-tools/action.yml | 6 +- .github/workflows/build_provider.yml | 26 +- .github/workflows/build_sdk.yml | 2 +- .github/workflows/community-moderation.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/master.yml | 269 +------------------ .github/workflows/nightly-test.yml | 75 +----- .github/workflows/prerelease.yml | 270 +------------------- .github/workflows/prerequisites.yml | 8 +- .github/workflows/pull-request.yml | 4 +- .github/workflows/release.yml | 270 +------------------- .github/workflows/run-acceptance-tests.yml | 284 +-------------------- 12 files changed, 60 insertions(+), 1158 deletions(-) diff --git a/.github/actions/setup-tools/action.yml b/.github/actions/setup-tools/action.yml index 862b4701f23..2d41dba14ae 100644 --- a/.github/actions/setup-tools/action.yml +++ b/.github/actions/setup-tools/action.yml @@ -20,7 +20,7 @@ runs: steps: - name: Install Go if: inputs.tools == 'all' || contains(inputs.tools, 'go') - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5 + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 with: go-version: "1.23.x" cache-dependency-path: | @@ -30,7 +30,7 @@ runs: - name: Install pulumictl if: inputs.tools == 'all' || contains(inputs.tools, 'pulumictl') - uses: jaxxstorm/action-install-gh-release@71d17cb091aa850acb2a1a4cf87258d183eb941b # v1.11.0 + uses: jaxxstorm/action-install-gh-release@cd6b2b78ad38bdd294341cda064ec0692b06215b # v1.14.0 with: tag: v0.0.46 repo: pulumi/pulumictl @@ -43,7 +43,7 @@ runs: - name: Install Schema Tools if: inputs.tools == 'all' || contains(inputs.tools, 'schema-tools') - uses: jaxxstorm/action-install-gh-release@71d17cb091aa850acb2a1a4cf87258d183eb941b # v1.11.0 + uses: jaxxstorm/action-install-gh-release@cd6b2b78ad38bdd294341cda064ec0692b06215b # v1.14.0 with: repo: pulumi/schema-tools diff --git a/.github/workflows/build_provider.yml b/.github/workflows/build_provider.yml index 75d30e38890..edfca3823b6 100644 --- a/.github/workflows/build_provider.yml +++ b/.github/workflows/build_provider.yml @@ -15,6 +15,7 @@ jobs: env: PROVIDER_VERSION: ${{ inputs.version }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AZURE_SIGNING_CONFIGURED: ${{ secrets.AZURE_SIGNING_CLIENT_ID != '' && secrets.AZURE_SIGNING_CLIENT_SECRET != '' && secrets.AZURE_SIGNING_TENANT_ID != '' && secrets.AZURE_SIGNING_KEY_VAULT_URI != '' }} strategy: fail-fast: true matrix: @@ -58,8 +59,31 @@ jobs: path: provider/cmd/pulumi-resource-aws - name: Restore makefile progress run: make --touch provider schema - - name: Build & package provider + + - name: Build provider + run: make "provider-${{ matrix.platform.os }}-${{ matrix.platform.arch }}" + + - name: Sign windows provider + if: matrix.platform.os == 'windows' && env.AZURE_SIGNING_CONFIGURED == 'true' + run: | + az login --service-principal \ + -u ${{ secrets.AZURE_SIGNING_CLIENT_ID }} \ + -p ${{ secrets.AZURE_SIGNING_CLIENT_SECRET }} \ + -t ${{ secrets.AZURE_SIGNING_TENANT_ID }} \ + -o none; + + wget https://github.com/ebourg/jsign/releases/download/6.0/jsign-6.0.jar; + + java -jar jsign-6.0.jar \ + --storetype AZUREKEYVAULT \ + --keystore "PulumiCodeSigning" \ + --url ${{ secrets.AZURE_SIGNING_KEY_VAULT_URI }} \ + --storepass "$(az account get-access-token --resource "https://vault.azure.net" | jq -r .accessToken)" \ + bin/windows-amd64/pulumi-resource-aws.exe; + + - name: Package provider run: make provider_dist-${{ matrix.platform.os }}-${{ matrix.platform.arch }} + - name: Upload artifacts uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: diff --git a/.github/workflows/build_sdk.yml b/.github/workflows/build_sdk.yml index 6aa92cddede..a4935bc1892 100644 --- a/.github/workflows/build_sdk.yml +++ b/.github/workflows/build_sdk.yml @@ -56,7 +56,7 @@ jobs: submodules: true persist-credentials: false - name: Cache examples generation - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 with: path: | .pulumi/examples-cache diff --git a/.github/workflows/community-moderation.yml b/.github/workflows/community-moderation.yml index a0a741509ea..f4dcc0a6178 100644 --- a/.github/workflows/community-moderation.yml +++ b/.github/workflows/community-moderation.yml @@ -26,7 +26,7 @@ jobs: - if: steps.sdk_changed.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name != github.repository name: Send codegen warning as comment on PR - uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: > diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 437c5dad780..050555d6ad4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,7 +38,7 @@ jobs: submodules: true persist-credentials: false - name: Install go - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5 + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5 with: # The versions of golangci-lint and setup-go here cross-depend and need to update together. go-version: 1.23 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index fdba7f7e878..46d9d95c874 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -99,10 +99,6 @@ jobs: - build_provider - test - license_check - - go_test_shim - - provider_test - - test_oidc - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: @@ -130,271 +126,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} test: - name: test + uses: ./.github/workflows/test.yml needs: - prerequisites - build_provider - build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - env: - PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }} - steps: - # Run as first step so we don't delete things that have just been installed - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - swap-storage: false - dotnet: false - - name: Checkout Repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: true - persist-credentials: false - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumicli, ${{ matrix.language }} - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - if: matrix.language == 'python' - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Make upstream - run: make upstream - - name: Run tests - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - go_test_shim: - name: Run test of provider shim - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go - - name: Prepare local workspace - run: make prepare_local_workspace - - name: go test - run: | - cd upstream - go get github.com/hashicorp/aws-sdk-go-base@v1.1.0 - cd shim - go test -v -coverprofile="coverage.txt" . - - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - timeout-minutes: 60 - provider_test: - name: provider_test - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node, dotnet, python, java - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Run provider tests - run: | - cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - test_oidc: - name: test_oidc - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - dotnet: ${{ matrix.language != 'dotnet' }} - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.4.0 - - name: Run selected tests with manual web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatchOidcManual -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - - name: Configure AWS Credentials for OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} - unset-current-credentials: true - - name: Run selected tests with configure-aws-credentials web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatch -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - upstream_lint: - name: Run upstream provider-lint - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - name: Install Go - uses: actions/setup-go@v5 - with: - cache: false - go-version: 1.23.x - - name: Prepare local workspace - run: make prepare_local_workspace - - name: upstream lint - run: | - cd upstream - make provider-lint - timeout-minutes: 60 + secrets: inherit + with: + version: ${{ needs.prerequisites.outputs.version }} name: master on: diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index f514c942252..c4228a9f71f 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -45,80 +45,15 @@ jobs: version: ${{ needs.prerequisites.outputs.version }} test: - name: test + uses: ./.github/workflows/test.yml needs: - prerequisites - build_provider - build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - env: - PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }} - steps: - # Run as first step so we don't delete things that have just been installed - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - swap-storage: false - dotnet: false - - name: Checkout Repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: true - persist-credentials: false - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumicli, ${{ matrix.language}} - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - if: matrix.language == 'python' - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Make upstream - run: make upstream - - name: Run tests - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java + secrets: inherit + with: + version: ${{ needs.prerequisites.outputs.version }} + name: cron on: schedule: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index b0d390a4731..1912cb4577e 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -59,10 +59,6 @@ jobs: - build_provider - test - license_check - - go_test_shim - - provider_test - - test_oidc - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: @@ -70,272 +66,14 @@ jobs: isPrerelease: true test: - name: test + uses: ./.github/workflows/test.yml needs: - prerequisites - build_provider - build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - env: - PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }} - steps: - # Run as first step so we don't delete things that have just been installed - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - swap-storage: false - dotnet: false - - name: Checkout Repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: true - persist-credentials: false - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumicli, nodejs, python, dotnet, go, java - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - if: matrix.language == 'python' - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Make upstream - run: make upstream - - name: Run tests - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - go_test_shim: - name: Run test of provider shim - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go - - name: Prepare local workspace - run: make prepare_local_workspace - - name: go test - run: | - cd upstream - go get github.com/hashicorp/aws-sdk-go-base@v1.1.0 - cd shim - go test -v -coverprofile="coverage.txt" . - - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - timeout-minutes: 60 - provider_test: - name: provider_test - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node, dotnet, python, java - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Run provider tests - run: | - cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - test_oidc: - name: test_oidc - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - dotnet: ${{ matrix.language != 'dotnet' }} - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.4.0 - - name: Run selected tests with manual web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatchOidcManual -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - - name: Configure AWS Credentials for OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} - unset-current-credentials: true - - name: Run selected tests with configure-aws-credentials web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatch -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - upstream_lint: - name: Run upstream provider-lint - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - name: Install Go - uses: actions/setup-go@v5 - with: - cache: false - go-version: 1.23.x - - name: Prepare local workspace - run: make prepare_local_workspace - - name: upstream lint - run: | - cd upstream - make provider-lint - timeout-minutes: 60 - + secrets: inherit + with: + version: ${{ needs.prerequisites.outputs.version }} name: prerelease on: diff --git a/.github/workflows/prerequisites.yml b/.github/workflows/prerequisites.yml index 694b3510b78..d48da63489f 100644 --- a/.github/workflows/prerequisites.yml +++ b/.github/workflows/prerequisites.yml @@ -62,7 +62,7 @@ jobs: with: set-env: 'PROVIDER_VERSION' - name: Cache examples generation - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 with: path: | .pulumi/examples-cache @@ -90,10 +90,10 @@ jobs: } >> "$GITHUB_ENV" - if: inputs.is_pr && inputs.is_automated == false name: Comment on PR with Details of Schema Check - uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - comment_tag: schemaCheck + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-tag: schemaCheck message: >+ ${{ env.SCHEMA_CHANGES }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 1bb3d6d6c2e..c68ace4e076 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -32,9 +32,9 @@ jobs: submodules: true persist-credentials: false - name: Comment PR - uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} message: > PR is now waiting for a maintainer to run the acceptance tests. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28b75d36f4e..e80c5c1b101 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,10 +65,6 @@ jobs: - build_provider - test - license_check - - go_test_shim - - provider_test - - test_oidc - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: @@ -76,269 +72,11 @@ jobs: isPrerelease: false test: - name: test + uses: ./.github/workflows/test.yml needs: - prerequisites - build_provider - build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - env: - PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }} - steps: - # Run as first step so we don't delete things that have just been installed - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - swap-storage: false - dotnet: false - - name: Checkout Repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: true - persist-credentials: false - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumicli, ${{ matrix.language }} - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - if: matrix.language == 'python' - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Make upstream - run: make upstream - - name: Run tests - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - go_test_shim: - name: Run test of provider shim - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go - - name: Prepare local workspace - run: make prepare_local_workspace - - name: go test - run: | - cd upstream - go get github.com/hashicorp/aws-sdk-go-base@v1.1.0 - cd shim - go test -v -coverprofile="coverage.txt" . - - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - timeout-minutes: 60 - provider_test: - name: provider_test - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node, dotnet, python, java - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Run provider tests - run: | - cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - test_oidc: - name: test_oidc - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - dotnet: ${{ matrix.language != 'dotnet' }} - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.4.0 - - name: Run selected tests with manual web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatchOidcManual -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - - name: Configure AWS Credentials for OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} - unset-current-credentials: true - - name: Run selected tests with configure-aws-credentials web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatch -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - upstream_lint: - name: Run upstream provider-lint - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - name: Install Go - uses: actions/setup-go@v5 - with: - cache: false - go-version: 1.23.x - - name: Prepare local workspace - run: make prepare_local_workspace - - name: upstream lint - run: | - cd upstream - make provider-lint - timeout-minutes: 60 - + secrets: inherit + with: + version: ${{ needs.prerequisites.outputs.version }} diff --git a/.github/workflows/run-acceptance-tests.yml b/.github/workflows/run-acceptance-tests.yml index 53b1bdae190..875adc2249a 100644 --- a/.github/workflows/run-acceptance-tests.yml +++ b/.github/workflows/run-acceptance-tests.yml @@ -94,10 +94,6 @@ jobs: - test - build_provider - license_check - - go_test_shim - - provider_test - - test_oidc - - upstream_lint runs-on: ubuntu-latest steps: - uses: guibranco/github-status-action-v2@0849440ec82c5fa69b2377725b9b7852a3977e76 # v1.1.13 @@ -113,287 +109,19 @@ jobs: sha: ${{ github.event.pull_request.head.sha || github.sha }} test: + # Don't run tests on PRs from forks. if: github.event_name == 'repository_dispatch' || github.event.pull_request.head.repo.full_name == github.repository - name: test + uses: ./.github/workflows/test.yml needs: - prerequisites - build_provider - build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - env: - PROVIDER_VERSION: ${{ needs.prerequisites.outputs.version }} - steps: - # Run as first step so we don't delete things that have just been installed - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - swap-storage: false - dotnet: false - - name: Checkout Repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - persist-credentials: false - - name: Checkout p/examples - if: matrix.testTarget == 'pulumiExamples' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: pulumi/examples - path: p-examples - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumicli, ${{ matrix.language }} - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - if: matrix.language == 'python' - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Make upstream - run: make upstream - - name: Run tests - if: matrix.testTarget == 'local' - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -skip TestPulumiExamples -parallel 4 . - - name: Run pulumi/examples tests - if: matrix.testTarget == 'pulumiExamples' - run: cd examples && go test -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -run TestPulumiExamples -parallel 4 . - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - testTarget: [local] + secrets: inherit + with: + version: ${{ needs.prerequisites.outputs.version }} + license_check: name: License Check uses: ./.github/workflows/license.yml secrets: inherit - go_test_shim: - name: Run test of provider shim - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go - - name: Prepare local workspace - run: make prepare_local_workspace - - name: go test - run: | - cd upstream - go get github.com/hashicorp/aws-sdk-go-base@v1.1.0 - cd shim - go test -v -coverprofile="coverage.txt" . - - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - timeout-minutes: 60 - provider_test: - name: provider_test - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node, dotnet, python, java - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install Python deps - run: |- - pip3 install virtualenv==20.0.23 - pip3 install pipenv - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.5.0 - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ env.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} - - name: Run provider tests - run: | - cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - - python - - dotnet - - go - - java - test_oidc: - name: test_oidc - needs: build_sdk - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - dotnet: ${{ matrix.language != 'dotnet' }} - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - uses: pulumi/provider-version-action@v1 - with: - set-env: PROVIDER_VERSION - - name: Setup tools - uses: ./.github/actions/setup-tools - with: - tools: pulumictl, pulumi, go, node - - name: Prepare local workspace - run: make prepare_local_workspace - - name: Download bin - uses: ./.github/actions/download-bin - - name: Download SDK - uses: ./.github/actions/download-sdk - with: - language: ${{ matrix.language }} - - name: Restore makefile progress - run: make --touch provider schema build_${{ matrix.language }} - - name: Update path - run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - - name: Install dependencies - run: make install_${{ matrix.language}}_sdk - - name: Install gotestfmt - uses: GoTestTools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - version: v2.4.0 - - name: Run selected tests with manual web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatchOidcManual -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - - name: Configure AWS Credentials for OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-duration-seconds: 3600 - role-session-name: aws@githubActions - role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} - unset-current-credentials: true - - name: Run selected tests with configure-aws-credentials web identity/OIDC auth - run: cd examples && go test -v -json -count=1 -run TestAccCloudWatch -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt - strategy: - fail-fast: false - matrix: - language: - - nodejs - upstream_lint: - name: Run upstream provider-lint - runs-on: ubuntu-latest - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - swap-storage: false - tool-cache: false - - name: Checkout Repo - uses: actions/checkout@v4 - with: - ref: ${{ env.PR_COMMIT_SHA }} - submodules: true - - name: Install Go - uses: actions/setup-go@v5 - with: - cache: false - go-version: 1.23.x - - name: Prepare local workspace - run: make prepare_local_workspace - - name: upstream lint - run: | - cd upstream - make provider-lint - timeout-minutes: 60 - From 012b61a50be5c7ead89efcaddadcbb178db3c627 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 11 Dec 2024 16:03:50 -0800 Subject: [PATCH 2/5] add missing file --- .github/workflows/test.yml | 340 +++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..a268324be52 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,340 @@ +name: "Test Provider" + +on: + workflow_call: + inputs: + version: + required: true + type: string + description: Version of the provider to test + +env: + PR_COMMIT_SHA: ${{ github.event.client_payload.pull_request.head.sha }} + AWS_REGION: us-west-2 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + NUGET_PUBLISH_KEY: ${{ secrets.NUGET_PUBLISH_KEY }} + OIDC_ROLE_ARN: ${{ secrets.OIDC_ROLE_ARN }} + PUBLISH_REPO_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} + PUBLISH_REPO_USERNAME: ${{ secrets.OSSRH_USERNAME }} + PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + PULUMI_API: https://api.pulumi-staging.io + PULUMI_GO_DEP_ROOT: ${{ github.workspace }}/.. + PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget + PULUMI_MISSING_DOCS_ERROR: "true" + PYPI_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + PYPI_USERNAME: __token__ + SIGNING_KEY: ${{ secrets.JAVA_SIGNING_KEY }} + SIGNING_KEY_ID: ${{ secrets.JAVA_SIGNING_KEY_ID }} + SIGNING_PASSWORD: ${{ secrets.JAVA_SIGNING_PASSWORD }} + TF_APPEND_USER_AGENT: pulumi + +jobs: + test: + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + env: + PROVIDER_VERSION: ${{ inputs.version }} + steps: + # Run as first step so we don't delete things that have just been installed + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: false + swap-storage: false + dotnet: false + - name: Checkout Repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ env.PR_COMMIT_SHA }} + submodules: true + persist-credentials: false + - name: Checkout p/examples + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: pulumi/examples + path: p-examples + - name: Setup tools + uses: ./.github/actions/setup-tools + - name: Prepare local workspace + run: make prepare_local_workspace + - name: Download bin + uses: ./.github/actions/download-bin + - name: Download nodejs SDK + uses: ./.github/actions/download-sdk + with: + language: nodejs + - name: Restore makefile progress + run: make --touch provider schema build_nodejs + - name: Install dependencies + run: make install_nodejs_sdk + - name: Download python SDK + uses: ./.github/actions/download-sdk + with: + language: python + - name: Restore makefile progress + run: make --touch provider schema build_python + - name: Install dependencies + run: make install_python_sdk + - name: Download dotnet SDK + uses: ./.github/actions/download-sdk + with: + language: dotnet + - name: Restore makefile progress + run: make --touch provider schema build_dotnet + - name: Install dependencies + run: make install_dotnet_sdk + - name: Download go SDK + uses: ./.github/actions/download-sdk + with: + language: go + - name: Restore makefile progress + run: make --touch provider schema build_go + - name: Install dependencies + run: make install_go_sdk + - name: Download java SDK + uses: ./.github/actions/download-sdk + with: + language: java + - name: Restore makefile progress + run: make --touch provider schema build_java + - name: Install dependencies + run: make install_java_sdk + - name: Update path + run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" + - name: Install Python deps + run: |- + pip3 install virtualenv==20.0.23 + pip3 install pipenv + - name: Install gotestfmt + uses: GoTestTools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: v2.5.0 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ env.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-session-name: aws@githubActions + role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} + - name: Make upstream + run: make upstream + - name: Generate shard + run: make shard + env: + SHARD_TOTAL: ${{ matrix.total }} + SHARD_INDEX: ${{ matrix.index }} + - name: Run integration tests + run: make test_shard + strategy: + fail-fast: false + matrix: + total: + - 10 + index: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + go_test_shim: + name: Run test of provider shim + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + swap-storage: false + tool-cache: false + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PR_COMMIT_SHA }} + submodules: true + - uses: pulumi/provider-version-action@v1 + with: + set-env: PROVIDER_VERSION + - name: Setup tools + uses: ./.github/actions/setup-tools + with: + tools: pulumictl, pulumi, go + - name: Prepare local workspace + run: make prepare_local_workspace + - name: go test + run: | + cd upstream + go get github.com/hashicorp/aws-sdk-go-base@v1.1.0 + cd shim + go test -v -coverprofile="coverage.txt" . + - env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + timeout-minutes: 60 + provider_test: + name: provider_test + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + swap-storage: false + tool-cache: false + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PR_COMMIT_SHA }} + submodules: true + - uses: pulumi/provider-version-action@v1 + with: + set-env: PROVIDER_VERSION + - name: Setup tools + uses: ./.github/actions/setup-tools + with: + tools: pulumictl, pulumi, go, node, dotnet, python, java + - name: Prepare local workspace + run: make prepare_local_workspace + - name: Download bin + uses: ./.github/actions/download-bin + - name: Download SDK + uses: ./.github/actions/download-sdk + with: + language: ${{ matrix.language }} + - name: Restore makefile progress + run: make --touch provider schema build_${{ matrix.language }} + - name: Update path + run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" + - name: Install Python deps + run: |- + pip3 install virtualenv==20.0.23 + pip3 install pipenv + - name: Install dependencies + run: make install_${{ matrix.language}}_sdk + - name: Install gotestfmt + uses: GoTestTools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: v2.5.0 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ env.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-session-name: aws@githubActions + role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }} + - name: Run provider tests + run: | + cd provider && go test -v -json -count=1 -cover -timeout 2h -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt + strategy: + fail-fast: false + matrix: + language: + - nodejs + - python + - dotnet + - go + - java + test_oidc: + name: test_oidc + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + dotnet: ${{ matrix.language != 'dotnet' }} + swap-storage: false + tool-cache: false + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PR_COMMIT_SHA }} + submodules: true + - uses: pulumi/provider-version-action@v1 + with: + set-env: PROVIDER_VERSION + - name: Setup tools + uses: ./.github/actions/setup-tools + with: + tools: pulumictl, pulumi, go, node + - name: Prepare local workspace + run: make prepare_local_workspace + - name: Download bin + uses: ./.github/actions/download-bin + - name: Download SDK + uses: ./.github/actions/download-sdk + with: + language: ${{ matrix.language }} + - name: Restore makefile progress + run: make --touch provider schema build_${{ matrix.language }} + - name: Update path + run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" + - name: Install dependencies + run: make install_${{ matrix.language}}_sdk + - name: Install gotestfmt + uses: GoTestTools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: v2.4.0 + - name: Run selected tests with manual web identity/OIDC auth + run: cd examples && go test -v -json -count=1 -run TestAccCloudWatchOidcManual -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt + - name: Configure AWS Credentials for OIDC + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-duration-seconds: 3600 + role-session-name: aws@githubActions + role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} + unset-current-credentials: true + - name: Run selected tests with configure-aws-credentials web identity/OIDC auth + run: cd examples && go test -v -json -count=1 -run TestAccCloudWatch -tags=${{ matrix.language }} -parallel 4 . 2>&1 | tee /tmp/gotest.log | gotestfmt + strategy: + fail-fast: false + matrix: + language: + - nodejs + upstream_lint: + name: Run upstream provider-lint + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + swap-storage: false + tool-cache: false + - name: Checkout Repo + uses: actions/checkout@v4 + with: + ref: ${{ env.PR_COMMIT_SHA }} + submodules: true + - name: Install Go + uses: actions/setup-go@v5 + with: + cache: false + go-version: 1.23.x + - name: Prepare local workspace + run: make prepare_local_workspace + - name: upstream lint + run: | + cd upstream + make provider-lint + timeout-minutes: 60 + From 553494c2bf7974e589062608be1d179d1de8cfcc Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 11 Dec 2024 16:40:45 -0800 Subject: [PATCH 3/5] revert windows signing --- .github/workflows/build_provider.yml | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/build_provider.yml b/.github/workflows/build_provider.yml index edfca3823b6..75d30e38890 100644 --- a/.github/workflows/build_provider.yml +++ b/.github/workflows/build_provider.yml @@ -15,7 +15,6 @@ jobs: env: PROVIDER_VERSION: ${{ inputs.version }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AZURE_SIGNING_CONFIGURED: ${{ secrets.AZURE_SIGNING_CLIENT_ID != '' && secrets.AZURE_SIGNING_CLIENT_SECRET != '' && secrets.AZURE_SIGNING_TENANT_ID != '' && secrets.AZURE_SIGNING_KEY_VAULT_URI != '' }} strategy: fail-fast: true matrix: @@ -59,31 +58,8 @@ jobs: path: provider/cmd/pulumi-resource-aws - name: Restore makefile progress run: make --touch provider schema - - - name: Build provider - run: make "provider-${{ matrix.platform.os }}-${{ matrix.platform.arch }}" - - - name: Sign windows provider - if: matrix.platform.os == 'windows' && env.AZURE_SIGNING_CONFIGURED == 'true' - run: | - az login --service-principal \ - -u ${{ secrets.AZURE_SIGNING_CLIENT_ID }} \ - -p ${{ secrets.AZURE_SIGNING_CLIENT_SECRET }} \ - -t ${{ secrets.AZURE_SIGNING_TENANT_ID }} \ - -o none; - - wget https://github.com/ebourg/jsign/releases/download/6.0/jsign-6.0.jar; - - java -jar jsign-6.0.jar \ - --storetype AZUREKEYVAULT \ - --keystore "PulumiCodeSigning" \ - --url ${{ secrets.AZURE_SIGNING_KEY_VAULT_URI }} \ - --storepass "$(az account get-access-token --resource "https://vault.azure.net" | jq -r .accessToken)" \ - bin/windows-amd64/pulumi-resource-aws.exe; - - - name: Package provider + - name: Build & package provider run: make provider_dist-${{ matrix.platform.os }}-${{ matrix.platform.arch }} - - name: Upload artifacts uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: From 8ec5cac8ffce7816eb9e7cffeaac91074d2c1a0d Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Wed, 11 Dec 2024 20:43:56 -0800 Subject: [PATCH 4/5] forgot the makefile --- Makefile | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 704106b5543..da7892ae4f5 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ GEN_ENVS := PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(G generate_dotnet: .make/generate_dotnet build_dotnet: .make/build_dotnet .make/generate_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_dotnet: bin/$(TFGEN) +.make/generate_dotnet: .make/install_plugins bin/$(TFGEN) $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/ cd sdk/dotnet/ && \ printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ @@ -109,7 +109,7 @@ build_dotnet: .make/build_dotnet generate_go: .make/generate_go build_go: .make/build_go .make/generate_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_go: bin/$(TFGEN) +.make/generate_go: .make/install_plugins bin/$(TFGEN) $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/ @touch $@ .make/build_go: .make/generate_go @@ -121,7 +121,7 @@ generate_java: .make/generate_java build_java: .make/build_java .make/generate_java: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) .make/generate_java: PACKAGE_VERSION := $(VERSION_GENERIC) -.make/generate_java: bin/pulumi-java-gen .make/schema +.make/generate_java: .make/install_plugins bin/pulumi-java-gen .make/schema PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(GEN_PULUMI_CONVERT_EXAMPLES_CACHE_DIR) bin/$(JAVA_GEN) generate --schema provider/cmd/$(PROVIDER)/schema.json --out sdk/java --build gradle-nexus printf "module fake_java_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/java/go.mod @touch $@ @@ -136,7 +136,7 @@ build_java: .make/build_java generate_nodejs: .make/generate_nodejs build_nodejs: .make/build_nodejs .make/generate_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_nodejs: bin/$(TFGEN) +.make/generate_nodejs: .make/install_plugins bin/$(TFGEN) $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/ printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/nodejs/go.mod @touch $@ @@ -151,7 +151,7 @@ build_nodejs: .make/build_nodejs generate_python: .make/generate_python build_python: .make/build_python .make/generate_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_python: bin/$(TFGEN) +.make/generate_python: .make/install_plugins bin/$(TFGEN) $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/ printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/python/go.mod cp README.md sdk/python/ @@ -326,6 +326,13 @@ bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: export CGO_ENABLED=0 && \ go build -o "${WORKING_DIR}/$@" $(PULUMI_PROVIDER_BUILD_PARALLELISM) -ldflags "$(LDFLAGS)" "$(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)" +provider-linux-amd64: bin/linux-amd64/$(PROVIDER) +provider-linux-arm64: bin/linux-arm64/$(PROVIDER) +provider-darwin-amd64: bin/darwin-amd64/$(PROVIDER) +provider-darwin-arm64: bin/darwin-arm64/$(PROVIDER) +provider-windows-amd64: bin/windows-amd64/$(PROVIDER).exe +.PHONY: provider-linux-amd64 provider-linux-arm64 provider-darwin-amd64 provider-darwin-arm64 provider-windows-amd64 + bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-amd64.tar.gz: bin/linux-amd64/$(PROVIDER) bin/$(PROVIDER)-v$(VERSION_GENERIC)-linux-arm64.tar.gz: bin/linux-arm64/$(PROVIDER) bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-amd64.tar.gz: bin/darwin-amd64/$(PROVIDER) @@ -344,6 +351,13 @@ provider_dist-darwin-arm64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-darwin-arm64.tar provider_dist-windows-amd64: bin/$(PROVIDER)-v$(VERSION_GENERIC)-windows-amd64.tar.gz provider_dist: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 .PHONY: provider_dist-linux-amd64 provider_dist-linux-arm64 provider_dist-darwin-amd64 provider_dist-darwin-arm64 provider_dist-windows-amd64 provider_dist - +# shard computes tests to run and modifies the CI runner's environment. +shard: + @(cd examples && go run github.com/blampe/shard@latest --total $(SHARD_TOTAL) --index $(SHARD_INDEX) --output env) >> "$(GITHUB_ENV)" + +# test_shard runs the tests specified by a regex contained in $SHARD_TESTS for paths $SHARD_PATHS. +test_shard: + cd examples && \ + go test -tags=all -v -count=1 -coverprofile="coverage.txt" -coverpkg=./... -timeout 3h -parallel ${TESTPARALLELISM} -run "$(SHARD_TESTS)" $(SHARD_PATHS) # Permit providers to extend the Makefile with provider-specific Make includes. include $(wildcard .mk/*.mk) From c6a4ddcc67ad63a95317621f710dd7c53549edf0 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Thu, 12 Dec 2024 15:19:53 -0800 Subject: [PATCH 5/5] fix dependencies --- .github/workflows/master.yml | 1 - .github/workflows/prerelease.yml | 1 - .github/workflows/release.yml | 1 - .github/workflows/run-acceptance-tests.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 217c710688a..46d9d95c874 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -99,7 +99,6 @@ jobs: - build_provider - test - license_check - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index ce99a9efe0f..1912cb4577e 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -59,7 +59,6 @@ jobs: - build_provider - test - license_check - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 493952b7a05..e80c5c1b101 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,6 @@ jobs: - build_provider - test - license_check - - upstream_lint uses: ./.github/workflows/publish.yml secrets: inherit with: diff --git a/.github/workflows/run-acceptance-tests.yml b/.github/workflows/run-acceptance-tests.yml index a38fbd03119..875adc2249a 100644 --- a/.github/workflows/run-acceptance-tests.yml +++ b/.github/workflows/run-acceptance-tests.yml @@ -94,7 +94,6 @@ jobs: - test - build_provider - license_check - - upstream_lint runs-on: ubuntu-latest steps: - uses: guibranco/github-status-action-v2@0849440ec82c5fa69b2377725b9b7852a3977e76 # v1.1.13