From 08e829320a56387432b2eaad7f8d118568c67f6f Mon Sep 17 00:00:00 2001 From: phlax Date: Tue, 6 Aug 2024 16:29:46 +0100 Subject: [PATCH] ci: Split build tests from release job (#35580) Signed-off-by: Ryan Northey --- .azure-pipelines/stage/linux.yml | 93 ---------------------------- .azure-pipelines/stage/prechecks.yml | 64 +++++++++++++++++-- .azure-pipelines/stage/publish.yml | 62 ++++++++++++++++++- .azure-pipelines/stages.yml | 40 ++---------- ci/do_ci.sh | 36 +++++++---- 5 files changed, 147 insertions(+), 148 deletions(-) delete mode 100644 .azure-pipelines/stage/linux.yml diff --git a/.azure-pipelines/stage/linux.yml b/.azure-pipelines/stage/linux.yml deleted file mode 100644 index ea465a768ae6..000000000000 --- a/.azure-pipelines/stage/linux.yml +++ /dev/null @@ -1,93 +0,0 @@ -parameters: -- name: cacheTestResults - displayName: "Cache test results" - type: boolean - default: true -- name: pool - displayName: "Agent pool" - type: string - default: envoy-x64-small -- name: artifactSuffix - displayName: "Artifact suffix" - type: string - default: -- name: runTests - displayName: "Run release tests" - type: string - default: true -- name: rbe - displayName: "Use RBE" - type: boolean - default: true -- name: timeoutBuild - displayName: "Build timeout" - type: number - default: 120 -- name: bazelBuildExtraOptions - type: string - default: "" -- name: bazelConfigRBE - type: string - default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs) - -- name: managedAgent - type: boolean - default: false -- name: tmpfsDockerDisabled - type: string - default: '' - - -- name: runBuild - displayName: "Run build" - type: string - default: true - -jobs: -- job: release - displayName: Build and test - condition: | - and(not(canceled()), - eq(${{ parameters.runBuild }}, 'true')) - timeoutInMinutes: ${{ parameters.timeoutBuild }} - pool: ${{ parameters.pool }} - steps: - - bash: | - if [[ "${{ parameters.runTests }}" == "false" ]]; then - CI_TARGET="release.server_only" - else - CI_TARGET="release" - fi - echo "${CI_TARGET}" - echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}" - name: target - - template: ../ci.yml - parameters: - artifactName: release - managedAgent: ${{ parameters.managedAgent }} - ciTarget: $(target.value) - cacheName: "release" - bazelConfigRBE: ${{ parameters.bazelConfigRBE }} - bazelBuildExtraOptions: ${{ parameters.bazelBuildExtraOptions }} - cacheTestResults: ${{ parameters.cacheTestResults }} - cacheVersion: $(cacheKeyBazel) - artifactSuffix: ${{ parameters.artifactSuffix }} - rbe: ${{ parameters.rbe }} - tmpfsDockerDisabled: ${{ parameters.tmpfsDockerDisabled }} - -- job: released - displayName: Complete - dependsOn: ["release"] - pool: - vmImage: $(agentUbuntu) - # This condition ensures that this (required) job passes if all of - # the preceeding jobs either pass or are skipped - # adapted from: - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage - condition: | - and(eq(variables['Build.Reason'], 'PullRequest'), - in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) - steps: - - checkout: none - - bash: | - echo "linux_x64 released" diff --git a/.azure-pipelines/stage/prechecks.yml b/.azure-pipelines/stage/prechecks.yml index b8bfaed62ab7..c4e2d779a6ba 100644 --- a/.azure-pipelines/stage/prechecks.yml +++ b/.azure-pipelines/stage/prechecks.yml @@ -25,6 +25,15 @@ parameters: type: string default: "" +- name: runBuild + displayName: "Run build" + type: string + default: true +- name: runPrechecks + displayName: "Run prechecks" + type: string + default: true + # Timeout/s - name: timeoutPrechecks type: number @@ -32,10 +41,10 @@ parameters: # a lot of change - eg protobuf changed, or a primitve proto changed. default: 40 -- name: runPrechecks - displayName: "Run prechecks" +- name: bazelConfigRBE type: string - default: true + default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs) + jobs: - job: prechecks @@ -53,17 +62,21 @@ jobs: matrix: format: CI_TARGET: "format" + CI_CACHE: format protobuf: CI_TARGET: "check_and_fix_proto_format" + CI_CACHE: check_and_fix_proto_format ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: publishing: CI_TARGET: docs + CI_CACHE: docs + steps: - template: ../ci.yml parameters: bazelBuildExtraOptions: --config=docs-ci ciTarget: $(CI_TARGET) - cacheName: $(CI_TARGET) + cacheName: $(CI_CACHE) cacheTestResults: ${{ parameters.cacheTestResults }} cacheVersion: $(cacheKeyBazel) publishEnvoy: false @@ -157,9 +170,47 @@ jobs: GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }} condition: eq(variables['CI_TARGET'], 'docs') +- job: precheck_release_x64 + displayName: Precheck release (x64) + condition: | + and(not(canceled()), + eq(${{ parameters.runBuild }}, 'true')) + timeoutInMinutes: 180 + pool: envoy-x64-large + steps: + - template: ../ci.yml + parameters: + artifactName: release + ciTarget: release.test_only + cacheName: release-test-only + bazelConfigRBE: ${{ parameters.bazelConfigRBE }} + cacheTestResults: ${{ parameters.cacheTestResults }} + cacheVersion: $(cacheKeyBazel) + rbe: true + +- job: precheck_release_arm64 + displayName: Precheck release (arm64) + condition: | + and(not(canceled()), + eq(${{ parameters.runBuild }}, 'true')) + timeoutInMinutes: 180 + pool: envoy-arm-large + steps: + - template: ../ci.yml + parameters: + artifactName: release + ciTarget: release.test_only + cacheName: release-test-only + bazelConfigRBE: ${{ parameters.bazelConfigRBE }} + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + cacheTestResults: ${{ parameters.cacheTestResults }} + cacheVersion: $(cacheKeyBazel) + artifactSuffix: .arm64 + rbe: false + - job: prechecked displayName: Prechecked - dependsOn: ["prechecks"] + dependsOn: ["prechecks", "precheck_release_arm64", "precheck_release_x64"] pool: vmImage: $(agentUbuntu) # This condition ensures that this (required) job passes if all of @@ -169,7 +220,8 @@ jobs: condition: | and( eq(variables['Build.Reason'], 'PullRequest'), - in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + in(dependencies.prechecks.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), + in(dependencies.precheck_release_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) steps: - checkout: none - bash: | diff --git a/.azure-pipelines/stage/publish.yml b/.azure-pipelines/stage/publish.yml index 284311240bf0..c81a8b416213 100644 --- a/.azure-pipelines/stage/publish.yml +++ b/.azure-pipelines/stage/publish.yml @@ -45,6 +45,14 @@ parameters: type: string default: "" +- name: bazelConfigRBE + type: string + default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs) + +- name: runBuild + displayName: "Run Build" + type: string + default: true - name: runDocker displayName: "Run Docker" type: string @@ -64,8 +72,58 @@ parameters: default: false jobs: + +- job: release_x64 + displayName: Binary release + condition: | + and(not(canceled()), + eq(${{ parameters.runBuild }}, 'true')) + timeoutInMinutes: 120 + pool: + vmImage: $(agentUbuntu) + steps: + - template: ../ci.yml + parameters: + artifactName: release + ciTarget: release.server_only + cacheName: release-server-only + bazelConfigRBE: ${{ parameters.bazelConfigRBE }} + cacheVersion: $(cacheKeyBazel) + cacheTestResults: false + publishTestResults: false + rbe: true + +- job: release_arm64 + displayName: Binary release (arm64) + condition: | + and(not(canceled()), + eq(${{ parameters.runBuild }}, 'true')) + timeoutInMinutes: 180 + pool: envoy-arm-large + steps: + - bash: | + CI_TARGET="release.server_only" + echo "${CI_TARGET}" + echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}" + name: target + - template: ../ci.yml + parameters: + artifactName: release + ciTarget: release.server_only + cacheName: release-server-only + bazelConfigRBE: ${{ parameters.bazelConfigRBE }} + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + cacheVersion: $(cacheKeyBazel) + cacheTestResults: false + artifactSuffix: .arm64 + publishTestResults: false + rbe: false + - job: docker displayName: "Docker (Linux multi arch)" + dependsOn: + - release_x64 + - release_arm64 condition: | and(not(canceled()), eq(${{ parameters.runDocker }}, 'true')) @@ -128,7 +186,7 @@ jobs: - job: package_x64 displayName: Linux debs (x64) - dependsOn: [] + dependsOn: ["release_x64"] condition: | and(not(canceled()), eq(${{ parameters.runPackaging }}, 'true')) @@ -162,7 +220,7 @@ jobs: - job: package_arm64 displayName: Linux debs (arm64) - dependsOn: [] + dependsOn: ["release_arm64"] condition: | and(not(canceled()), eq(${{ parameters.runPackaging }}, 'true')) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index f4ab9607aa06..a0fc0c9cbc1b 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -12,7 +12,7 @@ parameters: type: object default: - env - - linux_x64 + - prechecks - name: concurrencyChecks displayName: "Check concurrency" type: number @@ -59,40 +59,6 @@ stages: bucketGCP: $(GcsArtifactBucket) runPrechecks: stageDependencies.env.repo.outputs['run.releaseTests'] -- stage: linux_x64 - displayName: Linux x64 - dependsOn: ${{ parameters.buildStageDeps }} - variables: - RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] - RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']] - jobs: - - template: stage/linux.yml - parameters: - cacheTestResults: ${{ parameters.cacheTestResults }} - bazelConfigRBE: --config=remote-ci --config=rbe-google --jobs=200 - pool: envoy-x64-large - # these are parsed differently and _must_ be expressed in this way - runBuild: variables['RUN_BUILD'] - runTests: $(RUN_TESTS) - -- stage: linux_arm64 - displayName: Linux arm64 - dependsOn: ${{ parameters.buildStageDeps }} - variables: - RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] - RUN_TESTS: $[stageDependencies.env.repo.outputs['run.releaseTests']] - jobs: - - template: stage/linux.yml - parameters: - cacheTestResults: ${{ parameters.cacheTestResults }} - rbe: false - artifactSuffix: .arm64 - timeoutBuild: 180 - pool: envoy-arm-large - runBuild: variables['RUN_BUILD'] - runTests: $(RUN_TESTS) - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - - stage: check displayName: Checks (Linux x64) dependsOn: ${{ parameters.checkStageDeps }} @@ -110,8 +76,9 @@ stages: - stage: publish displayName: Publish - dependsOn: ["env", "linux_x64", "linux_arm64"] + dependsOn: ["env", "prechecks"] variables: + RUN_BUILD: $[stageDependencies.env.repo.outputs['run.build']] RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] RUN_PACKAGING: $[stageDependencies.env.repo.outputs['run.packaging']] PUBLISH_GITHUB_RELEASE: $[stageDependencies.env.repo.outputs['publish.githubRelease']] @@ -131,6 +98,7 @@ stages: bucketGCP: $(GcsArtifactBucket) timeoutDockerBuild: ${{ parameters.timeoutDockerBuild }} timeoutDockerPublish: ${{ parameters.timeoutDockerPublish }} + runBuild: variables['RUN_BUILD'] runDocker: variables['RUN_DOCKER'] runPackaging: variables['RUN_PACKAGING'] publishDockerhub: variables['PUBLISH_DOCKERHUB'] diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e500dd8792a4..91f7ebb047b4 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -238,7 +238,7 @@ if [[ $# -ge 1 ]]; then else # Coverage test will add QUICHE tests by itself. COVERAGE_TEST_TARGETS=("//test/...") - if [[ "${CI_TARGET}" == "release" ]]; then + if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then # We test contrib on release only. COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") elif [[ "${CI_TARGET}" == "msan" ]]; then @@ -700,11 +700,15 @@ case $CI_TARGET in fetch-gcc) targets=("${FETCH_GCC_TARGETS[@]}") ;; - fetch-release) + fetch-release|fetch-release.test_only) targets=( "${FETCH_BUILD_TARGETS[@]}" "${FETCH_ALL_TEST_TARGETS[@]}") ;; + fetch-release.server_only) + targets=( + "${FETCH_BUILD_TARGETS[@]}") + ;; fetch-*coverage) targets=("${FETCH_TEST_TARGETS[@]}") ;; @@ -807,8 +811,8 @@ case $CI_TARGET in "${PUBLISH_ARGS[@]}" ;; - release|release.server_only) - if [[ "$CI_TARGET" == "release" ]]; then + release|release.server_only|release.test_only) + if [[ "$CI_TARGET" == "release" || "$CI_TARGET" == "release.test_only" ]]; then # When testing memory consumption, we want to test against exact byte-counts # where possible. As these differ between platforms and compile options, we # define the 'release' builds as canonical and test them only in CI, so the @@ -820,19 +824,13 @@ case $CI_TARGET in fi fi setup_clang_toolchain - ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" - if [[ -e "${ENVOY_BINARY_DIR}" ]]; then - echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." - rm -rf "${ENVOY_BINARY_DIR}" - fi - mkdir -p "$ENVOY_BINARY_DIR" # As the binary build package enforces compiler options, adding here to ensure the tests and distribution build # reuse settings and any already compiled artefacts, the bundle itself will always be compiled # `--stripopt=--strip-all -c opt` BAZEL_RELEASE_OPTIONS=( --stripopt=--strip-all -c opt) - if [[ "$CI_TARGET" == "release" ]]; then + if [[ "$CI_TARGET" == "release" || "$CI_TARGET" == "release.test_only" ]]; then # Run release tests echo "Testing with:" echo " targets: ${TEST_TARGETS[*]}" @@ -844,6 +842,22 @@ case $CI_TARGET in "${BAZEL_RELEASE_OPTIONS[@]}" \ "${TEST_TARGETS[@]}" fi + + if [[ "$CI_TARGET" == "release.test_only" ]]; then + exit 0 + fi + + ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" + if [[ -e "${ENVOY_BINARY_DIR}" ]]; then + echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." + rm -rf "${ENVOY_BINARY_DIR}" + fi + mkdir -p "$ENVOY_BINARY_DIR" + + # Build + echo "Building with:" + echo " release options: ${BAZEL_RELEASE_OPTIONS[*]}" + # Build release binaries bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ "${BAZEL_RELEASE_OPTIONS[@]}" \