repo/ci: Shift deb verification -> github (envoyproxy#35579)

Signed-off-by: Ryan Northey <ryan@synca.io> Signed-off-by: phlax <phlax@users.noreply.github.com>
phlax · Aug 22, 2024 · 2bb4cf1 · 2bb4cf1
1 parent 08e8293
commit 2bb4cf1
Show file tree

Hide file tree

Showing 7 changed files with 177 additions and 206 deletions.
diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml
diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml
@@ -103,13 +103,3 @@ stages:
  runPackaging: variables['RUN_PACKAGING']
  publishDockerhub: variables['PUBLISH_DOCKERHUB']
  publishGithubRelease: variables['PUBLISH_GITHUB_RELEASE']
-
-- stage: verify
- displayName: Verify
- dependsOn: ["env", "publish"]
- variables:
- RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']]
- jobs:
- - template: stage/verify.yml
- parameters:
- authGCP: $(GcpServiceAccountKey)
diff --git a/.github/workflows/_stage_publish.yml → .github/workflows/_publish_publish.yml b/.github/workflows/_stage_publish.yml → .github/workflows/_publish_publish.yml
diff --git a/.github/workflows/_publish_verify.yml b/.github/workflows/_publish_verify.yml
@@ -0,0 +1,166 @@
+name: Verify
+
+permissions:
+ contents: read
+
+on:
+ workflow_call:
+ inputs:
+ request:
+ type: string
+ required: true
+ trusted:
+ type: boolean
+ required: true
+
+concurrency:
+ group: >-
+ ${{ github.actor != 'trigger-release-envoy[bot]'
+ && github.event.inputs.head_ref
+ || github.run_id
+ }}-${{ github.event.workflow.id }}-verify
+ cancel-in-progress: true
+
+
+jobs:
+ verify-examples:
+ permissions:
+ contents: read
+ packages: read
+ name: ${{ matrix.name || matrix.target }}
+ uses: ./.github/workflows/_run.yml
+ with:
+ bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }}
+ cache-build-image: ${{ matrix.cache-build-image }}
+ cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }}
+ container-command: ${{ matrix.container-command }}
+ concurrency-suffix: -${{ matrix.arch || 'x64' }}
+ rbe: ${{ matrix.rbe }}
+ request: ${{ inputs.request }}
+ runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }}
+ steps-pre: ${{ matrix.steps-pre }}
+ source: ${{ matrix.source }}
+ target: ${{ matrix.target }}
+ trusted: ${{ inputs.trusted }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: examples
+ target: verify_examples
+ rbe: false
+ source: |
+ export NO_BUILD_SETUP=1
+ steps-pre: |
+ - run: |
+ # Install expected host packages
+ export DEBIAN_FRONTEND=noninteractive
+ sudo apt-get -qq update -y
+ sudo apt-get -qq install -y --no-install-recommends expect gettext yq whois
+ shell: bash
+ - id: url
+ uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.35
+ with:
+ options: -Rr
+ input: >-
+ ${{ inputs.trusted
+ && fromJSON(inputs.request).request.sha
+ || fromJSON(inputs.request).request.ref }}
+ filter: |
+ .[:7] as $sha
+ | if ${{ inputs.trusted }} then
+ "envoy-postsubmit"
+ else
+ "envoy-pr"
+ end
+ | . as $bucket
+ | "https://storage.googleapis.com/\($bucket)/\($sha)"
+ - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35
+ with:
+ url: %{{ steps.url.outputs.value }}/docker/envoy.tar
+ variant: dev
+ - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35
+ with:
+ url: %{{ steps.url.outputs.value }}/docker/envoy-contrib.tar
+ variant: contrib-dev
+ - uses: envoyproxy/toolshed/gh-actions/docker/fetch@actions-v0.2.35
+ with:
+ url: %{{ steps.url.outputs.value }}/docker/envoy-google-vrp.tar
+ variant: google-vrp-dev
+ - run: docker images | grep envoy
+ shell: bash
+
+ verify-distro:
+ permissions:
+ contents: read
+ packages: read
+ name: ${{ matrix.name || matrix.target }}
+ uses: ./.github/workflows/_run.yml
+ with:
+ bazel-extra: ${{ matrix.bazel-extra || '--config=rbe-envoy-engflow' }}
+ cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
+ cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }}
+ container-command: ./ci/run_envoy_docker.sh
+ concurrency-suffix: -${{ matrix.arch || 'x64' }}
+ rbe: ${{ matrix.rbe && matrix.rbe || false }}
+ request: ${{ inputs.request }}
+ runs-on: ${{ matrix.runs-on || 'ubuntu-24.04' }}
+ source: |
+ export NO_BUILD_SETUP=1
+ export ENVOY_DOCKER_IN_DOCKER=1
+ target: ${{ matrix.target }}
+ trusted: ${{ inputs.trusted }}
+ steps-pre: |
+ - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.2.30
+ id: url
+ with:
+ options: -Rr
+ input: >-
+ ${{ inputs.trusted
+ && fromJSON(inputs.request).request.sha
+ || fromJSON(inputs.request).request.ref }}
+ filter: |
+ .[:7] as $sha
+ | if ${{ inputs.trusted }} then
+ "envoy-postsubmit"
+ else
+ "envoy-pr"
+ end
+ | . as $bucket
+ | "https://storage.googleapis.com/\($bucket)/\($sha)/release/release.signed.tar.zst"
+ - uses: envoyproxy/toolshed/gh-actions/fetch@actions-v0.2.30
+ id: fetch
+ with:
+ url: %{{ steps.url.outputs.value }}
+ - run: |
+ echo ARCH=${{ matrix.arch || 'x64' }} >> $GITHUB_ENV
+ echo DEB_ARCH=${{ matrix.arch != 'arm64' && 'amd64' || 'arm64' }} >> $GITHUB_ENV
+ shell: bash
+ - run: |
+ TEMP_DIR=$(mktemp -d)
+ zstd --stdout -d %{{ steps.fetch.outputs.path }} | tar --warning=no-timestamp -xf - -C "${TEMP_DIR}"
+ mkdir ${TEMP_DIR}/debs
+ tar xf ${TEMP_DIR}/bin/debs.tar.gz -C ${TEMP_DIR}/debs
+ mkdir -p ${TEMP_DIR}/distribution/deb
+ cp -a ${TEMP_DIR}/debs/*_${DEB_ARCH}* ${TEMP_DIR}/distribution/deb
+ cp -a ${TEMP_DIR}/signing.key ${TEMP_DIR}/distribution
+ mkdir -p %{{ runner.temp }}/distribution/${ARCH}
+ tar czf %{{ runner.temp }}/distribution/${ARCH}/packages.${ARCH}.tar.gz -C ${TEMP_DIR}/distribution .
+ shell: bash
+
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+
+ - name: verify_distro_x64
+ target: verify_distro
+ rbe: true
+
+ - name: verify_distro_arm64
+ target: verify_distro
+ arch: arm64
+ bazel-extra: >-
+ --config=cache-envoy-engflow
+ --config=bes-envoy-engflow
+ runs-on: envoy-arm64-small
diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml
@@ -21,11 +21,16 @@ on:
  default: 75
  cache-build-image:
  type: string
+ cache-build-image-key-suffix:
+ type: string
  catch-errors:
  type: boolean
  default: false
  checkout-extra:
  type: string
+ concurrency-suffix:
+ type: string
+ default:
  container-command:
  type: string
  default: ./ci/run_envoy_docker.sh
@@ -140,7 +145,7 @@ concurrency:
  ${{ github.actor != 'trigger-release-envoy[bot]'
  && github.head_ref
  || github.run_id
- }}-${{ github.workflow }}-${{ inputs.target }}
+ }}-${{ github.workflow }}-${{ inputs.target }}${{ inputs.concurrency-suffix }}
  cancel-in-progress: true
 
 env:
@@ -189,6 +194,7 @@ jobs:
  uses: envoyproxy/toolshed/gh-actions/docker/cache/restore@actions-v0.2.30
  with:
  image_tag: ${{ inputs.cache-build-image }}
+ key-suffix: ${{ inputs.cache-build-image-key-suffix }}
 
  - uses: envoyproxy/toolshed/gh-actions/appauth@actions-v0.2.30
  id: appauth
@@ -258,11 +264,11 @@ jobs:
  env:
  GITHUB_TOKEN: ${{ inputs.trusted && steps.appauth.outputs.token || github.token }}
  ENVOY_DOCKER_BUILD_DIR: ${{ runner.temp }}
- ENVOY_RBE: ${{ inputs.rbe != 'false' && 1 || '' }}
+ ENVOY_RBE: ${{ inputs.rbe == true && 1 || '' }}
  RBE_KEY: ${{ secrets.rbe-key }}
  BAZEL_BUILD_EXTRA_OPTIONS: >-
  --config=remote-ci
  ${{ inputs.bazel-extra }}
- ${{ inputs.rbe != 'false' && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }}
+ ${{ inputs.rbe == true && format('--jobs={0}', inputs.bazel-rbe-jobs) || '' }}
  BAZEL_FAKE_SCM_REVISION: ${{ github.event_name == 'pull_request' && 'e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9' || '' }}
  CI_TARGET_BRANCH: ${{ fromJSON(inputs.request).request.target-branch }}