diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml deleted file mode 100644 index 20bff07dc7cc..000000000000 --- a/.azure-pipelines/stage/verify.yml +++ /dev/null @@ -1,84 +0,0 @@ -parameters: - -# Auth -- name: authGCP - type: string - default: "" - - -jobs: -- job: packages_x64 - displayName: Debs (x64) - condition: | - and(not(canceled()), - succeeded(), - ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), - ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), - ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) - timeoutInMinutes: 120 - pool: envoy-x64-small - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "distribution" - itemPattern: "distribution/x64/packages.x64.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - template: ../ci.yml - parameters: - ciTarget: verify_distro - cacheName: verify_distro - publishTestResults: false - tmpfsDockerDisabled: true - env: - ENVOY_DOCKER_IN_DOCKER: 1 - -- job: packages_arm64 - displayName: Debs (arm64) - condition: | - and(not(canceled()), - succeeded(), - ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), - ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), - ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) - timeoutInMinutes: 120 - pool: "envoy-arm-small" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "distribution" - itemPattern: "distribution/arm64/packages.arm64.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - template: ../ci.yml - parameters: - managedAgent: false - ciTarget: verify_distro - cacheName: verify_distro - rbe: false - artifactSuffix: ".arm64" - publishTestResults: false - tmpfsDockerDisabled: true - env: - ENVOY_DOCKER_IN_DOCKER: 1 - -- job: verified - displayName: Verification complete - dependsOn: ["packages_x64", "packages_arm64"] - pool: - vmImage: $(agentUbuntu) - # This condition ensures that this (required) check passes if all of - # the preceding checks either pass or are skipped - # adapted from: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage - condition: | - and( - eq(variables['Build.Reason'], 'PullRequest'), - in(dependencies.packages_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), - in(dependencies.packages_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - steps: - - checkout: none - - bash: | - echo "checks complete" diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index a0fc0c9cbc1b..72438bf0ab46 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -103,13 +103,3 @@ stages: runPackaging: variables['RUN_PACKAGING'] publishDockerhub: variables['PUBLISH_DOCKERHUB'] publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE'] - -- stage: verify - displayName: Verify - dependsOn: ["env", "publish"] - variables: - RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] - jobs: - - template: stage/verify.yml - parameters: - authGCP: $(GcpServiceAccountKey) diff --git a/.github/workflows/_stage_publish.yml b/.github/workflows/_publish_publish.yml similarity index 100% rename from .github/workflows/_stage_publish.yml rename to .github/workflows/_publish_publish.yml diff --git a/.github/workflows/_publish_verify.yml b/.github/workflows/_publish_verify.yml new file mode 100644 index 000000000000..075e4aad0440 --- /dev/null +++ b/.github/workflows/_publish_verify.yml @@ -0,0 +1,166 @@ +name: Verify + +permissions: + contents: read + +on: + workflow_call: + inputs: + request: + type: string + required: true + trusted: + type: boolean + required: true + +concurrency: + group: >- + ${{ github.actor != 'trigger-release-envoy[bot]' + && github.event.inputs.head_ref + || github.run_id + }}-${{ github.event.workflow.id }}-verify + cancel-in-progress: true + + +jobs: + verify-examples: + permissions: + contents: read + packages: read + name: ${{ matrix.name || matrix.target }} + uses: ./.github/workflows/_run.yml + with: + bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }} + cache-build-image: ${{ matrix.cache-build-image }} + cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }} + container-command: ${{ matrix.container-command }} + concurrency-suffix: -${{ matrix.arch || 'x64' }} + rbe: ${{ matrix.rbe }} + request: ${{ inputs.request }} + runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }} + steps-pre: ${{ matrix.steps-pre }} + source: ${{ matrix.source }} + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} + strategy: + fail-fast: false + matrix: + include: + - name: examples + target: verify_examples + rbe: false + source: | + export NO_BUILD_SETUP=1 + steps-pre: | + - run: | + # Install expected host packages + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qq update -y + sudo apt-get -qq install -y --no-install-recommends expect gettext yq whois + shell: bash + - id: url + uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.35 + with: + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)" + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy.tar + variant: dev + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar + variant: contrib-dev + - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35 + with: + url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar + variant: google-vrp-dev + - run: docker images | grep envoy + shell: bash + + verify-distro: + permissions: + contents: read + packages: read + name: ${{ matrix.name || matrix.target }} + uses: ./.github/workflows/_run.yml + with: + bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }} + cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }} + cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }} + container-command: ./ci/run_envoy_docker.sh + concurrency-suffix: -${{ matrix.arch || 'x64' }} + rbe: ${{ matrix.rbe && matrix.rbe || false }} + request: ${{ inputs.request }} + runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }} + source: | + export NO_BUILD_SETUP=1 + export ENVOY_DOCKER_IN_DOCKER=1 + target: ${{ matrix.target }} + trusted: ${{ inputs.trusted }} + steps-pre: | + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.30 + id: url + with: + options: -Rr + input: >- + ${{ inputs.trusted + && fromJSON(inputs.request).request.sha + || fromJSON(inputs.request).request.ref }} + filter: | + .[:7] as $sha + | if ${{ inputs.trusted }} then + "envoy-postsubmit" + else + "envoy-pr" + end + | . as $bucket + | "https://storage.googleapis.com/\($bucket)/\($sha)/release/release.signed.tar.zst" + - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.2.30 + id: fetch + with: + url: %{{ steps.url.outputs.value }} + - run: | + echo ARCH=${{ matrix.arch || 'x64' }} >> $GITHUB_ENV + echo DEB_ARCH=${{ matrix.arch != 'arm64' && 'amd64' || 'arm64' }} >> $GITHUB_ENV + shell: bash + - run: | + TEMP_DIR=$(mktemp -d) + zstd --stdout -d %{{ steps.fetch.outputs.path }} | tar --warning=no-timestamp -xf - -C "${TEMP_DIR}" + mkdir ${TEMP_DIR}/debs + tar xf ${TEMP_DIR}/bin/debs.tar.gz -C ${TEMP_DIR}/debs + mkdir -p ${TEMP_DIR}/distribution/deb + cp -a ${TEMP_DIR}/debs/*_${DEB_ARCH}* ${TEMP_DIR}/distribution/deb + cp -a ${TEMP_DIR}/signing.key ${TEMP_DIR}/distribution + mkdir -p %{{ runner.temp }}/distribution/${ARCH} + tar czf %{{ runner.temp }}/distribution/${ARCH}/packages.${ARCH}.tar.gz -C ${TEMP_DIR}/distribution . + shell: bash + + strategy: + fail-fast: false + matrix: + include: + + - name: verify_distro_x64 + target: verify_distro + rbe: true + + - name: verify_distro_arm64 + target: verify_distro + arch: arm64 + bazel-extra: >- + --config=cache-envoy-engflow + --config=bes-envoy-engflow + runs-on: envoy-arm64-small diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 92620c5325d7..dbfaa259d8a7 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -21,11 +21,16 @@ on: default: 75 cache-build-image: type: string + cache-build-image-key-suffix: + type: string catch-errors: type: boolean default: false checkout-extra: type: string + concurrency-suffix: + type: string + default: container-command: type: string default: ./ci/run_envoy_docker.sh @@ -140,7 +145,7 @@ concurrency: ${{ github.actor != 'trigger-release-envoy[bot]' && github.head_ref || github.run_id - }}-${{ github.workflow }}-${{ inputs.target }} + }}-${{ github.workflow }}-${{ inputs.target }}${{ inputs.concurrency-suffix }} cancel-in-progress: true env: @@ -189,6 +194,7 @@ jobs: uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.2.30 with: image_tag: ${{ inputs.cache-build-image }} + key-suffix: ${{ inputs.cache-build-image-key-suffix }} - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.30 id: appauth @@ -258,11 +264,11 @@ jobs: env: GITHUB_TOKEN: ${{ inputs.trusted && steps.appauth.outputs.token || github.token }} ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }} - ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }} + ENVOY_RBE: ${{ inputs.rbe == true && 1 || '' }} RBE_KEY: ${{ secrets.rbe-key }} BAZEL_BUILD_EXTRA_OPTIONS: >- --config=remote-ci ${{ inputs.bazel-extra }} - ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} + ${{ inputs.rbe == true && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }} BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }} CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }} diff --git a/.github/workflows/_stage_verify.yml b/.github/workflows/_stage_verify.yml deleted file mode 100644 index c6de8a7c1bf6..000000000000 --- a/.github/workflows/_stage_verify.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Verify - -permissions: - contents: read - -on: - workflow_call: - inputs: - request: - type: string - required: true - trusted: - type: boolean - required: true - -concurrency: - group: >- - ${{ github.actor != 'trigger-release-envoy[bot]' - && github.event.inputs.head_ref - || github.run_id - }}-${{ github.event.workflow.id }}-verify - cancel-in-progress: true - - -jobs: - verify: - permissions: - contents: read - packages: read - name: ${{ matrix.name || matrix.target }} - uses: ./.github/workflows/_run.yml - with: - cache-build-image: - container-command: - rbe: ${{ matrix.rbe }} - request: ${{ inputs.request }} - runs-on: envoy-x64-small - steps-pre: ${{ matrix.steps-pre }} - source: ${{ matrix.source }} - target: ${{ matrix.target }} - trusted: ${{ inputs.trusted }} - strategy: - fail-fast: false - matrix: - include: - - name: examples - target: verify_examples - source: | - export NO_BUILD_SETUP=1 - rbe: false - steps-pre: | - - run: | - # TODO(phlax): Remove this once resolved properly - # Downgrade Docker to workaround https://github.com/containers/skopeo/issues/2365 - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ - https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ - | sudo tee /etc/apt/sources.list.d/docker.list \ - > /dev/null - sudo apt-get update - sudo apt-get install -y -qq --allow-downgrades \ - docker-ce=5:24.0.9-1~ubuntu.22.04~jammy \ - docker-ce-cli=5:24.0.9-1~ubuntu.22.04~jammy - sudo systemctl restart docker - sudo docker --version - shell: bash - - id: url - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.30 - with: - options: -Rr - input: >- - ${{ inputs.trusted - && fromJSON(inputs.request).request.sha - || fromJSON(inputs.request).request.ref }} - filter: | - .[:7] as $sha - | if ${{ inputs.trusted }} then - "envoy-postsubmit" - else - "envoy-pr" - end - | . as $bucket - | "https://storage.googleapis.com/\($bucket)/\($sha)" - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.30 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy.tar - variant: dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.30 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar - variant: contrib-dev - - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.30 - with: - url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar - variant: google-vrp-dev - - run: docker images | grep envoy - shell: bash - - run: | - # Install expected host packages - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect gettext whois - pip install -r ./.github/workflows/verify-requirements.txt - shell: bash diff --git a/.github/workflows/envoy-publish.yml b/.github/workflows/envoy-publish.yml index ab7a7b896292..df33cd5221ba 100644 --- a/.github/workflows/envoy-publish.yml +++ b/.github/workflows/envoy-publish.yml @@ -62,7 +62,7 @@ jobs: if: ${{ fromJSON(needs.load.outputs.request).run.publish }} needs: - load - uses: ./.github/workflows/_stage_publish.yml + uses: ./.github/workflows/_publish_publish.yml name: Publish with: request: ${{ needs.load.outputs.request }} @@ -75,7 +75,7 @@ jobs: if: ${{ fromJSON(needs.load.outputs.request).run.verify }} needs: - load - uses: ./.github/workflows/_stage_verify.yml + uses: ./.github/workflows/_publish_verify.yml name: Verify with: request: ${{ needs.load.outputs.request }}