pipelines.yml

trigger:
  branches:
    include:
    - "main"
    - "release/v*"
  tags:
    include:
    - "v*"

# PR build config is manually overridden in Azure pipelines UI with different secrets
pr: none

variables:
- name: isDev
  # This must be checked/set in a `step` from the VERSION.txt file, in order to be useful
  value: true
- name: isMain
  value: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
- name: isReleaseBranch
  value: $[startsWith(variables['Build.SourceBranch'], 'refs/heads/release/v')]
- name: isTaggedRelease
  value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/v')]
- name: isStableBranch
  # A release branch can be either `main` or a `release/v1.x` stable branch
  value: $[or(eq(variables['isMain'], 'true'), eq(variables['isReleaseBranch'], 'true'))]

stages:
- stage: env
  #
  # Note to use the outputs created here - this `stage` must be set as a dependency in the relevant stage
  # eg:
  #
  # stage: mystage
  # dependsOn: ["env"]
  #
  # To use a variable as a condition for a `stage` you can specify it as follows:
  #
  # condition: and(not(canceled()), succeeded(), ne(dependencies.env.outputs['repo.changed.mobileOnly'], 'true'))
  #
  # To use a variable as a condition for a `job` you can specify it as follows:
  #
  # job:
  #   condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'))
  #
  # Note the difference in name resolution when used in stages and jobs.
  #
  # Note also, other conditions such as `not(canceled())` are not assumed and so need to be added to
  # any conditions explicitly.
  #
  # NB: Avoid using _inclusive_ lists to trigger CI - eg _dont_ only trigger mobile CI if `mobile/` changed.
  #  Instead use _exclusive_ bools - eg _do_ suppress other CI if only `mobile/` changed
  #
  # To use a variable in a `step` you can specify it as follows:
  #
  # stage: my_next_stage
  #   jobs:
  #     job: another_job
  #     variables:
  #       mobileOnly: $[stageDependencies.env.repo.outputs['changed.mobileOnly']]
  #     steps:
  #     - bash: echo $(mobileOnly)
  #

  jobs:
  - job: repo
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - checkout: self
      fetchDepth: 0
      fetchTags: true
    - bash: |
        set -e

        # Only exclude checks in pull requests.
        if [[ $(Build.Reason) != "PullRequest" ]]; then
            echo "##vso[task.setvariable variable=mobileOnly;isoutput=true]false"
            echo "##vso[task.setvariable variable=docsOnly;isoutput=true]false"
            echo "##vso[task.setvariable variable=examplesOnly;isoutput=true]false"
            exit 0
        fi

        CHANGE_TARGET="origin/$(System.PullRequest.TargetBranch)"

        echo "Comparing changes ${CHANGE_TARGET}...HEAD"
        CHANGED_PATHS="$(git diff --name-only ${CHANGE_TARGET}...HEAD | cut -d/ -f1 | sort -u | jq -sR 'rtrimstr("\n") | split("\n")')"
        echo "$CHANGED_PATHS" | jq '.'
        CHANGED_PATH_COUNT=$(echo $CHANGED_PATHS | jq '. | length')
        CHANGED_MOBILE_ONLY=false
        CHANGED_DOCS_ONLY=false
        CHANGED_EXAMPLES_ONLY=false

        CHANGED_MOBILE=$(echo $CHANGED_PATHS | jq '. as $A | "mobile" | IN($A[])')
        if [[ $CHANGED_MOBILE == true && $CHANGED_PATH_COUNT -eq 1 ]]; then
            CHANGED_MOBILE_ONLY=true
        fi
        CHANGED_DOCS=$(echo $CHANGED_PATHS | jq '. as $A | "docs" | IN($A[])')
        if [[ $CHANGED_DOCS == true && $CHANGED_PATH_COUNT -eq 1 ]]; then
            CHANGED_DOCS_ONLY=true
        fi
        CHANGED_EXAMPLES=$(echo $CHANGED_PATHS | jq '. as $A | "examples" | IN($A[])')
        if [[ $CHANGED_EXAMPLES == true && $CHANGED_PATH_COUNT -eq 1 ]]; then
            CHANGED_EXAMPLES_ONLY=true
        fi

        echo "##vso[task.setvariable variable=mobileOnly;isoutput=true]${CHANGED_MOBILE_ONLY}"
        echo "##vso[task.setvariable variable=docsOnly;isoutput=true]${CHANGED_DOCS_ONLY}"
        echo "##vso[task.setvariable variable=examplesOnly;isoutput=true]${CHANGED_EXAMPLES_ONLY}"
      displayName: "Detect repo changes"
      workingDirectory: $(Build.SourcesDirectory)
      name: changed
    - bash: |
        echo "env.outputs['changed.mobileOnly']: $(changed.mobileOnly)"
        echo "env.outputs['changed.docsOnly']: $(changed.docsOnly)"
        echo "env.outputs['changed.examplesOnly']: $(changed.examplesOnly)"
      displayName: "Print build environment"

- stage: precheck
  dependsOn: ["env"]
  jobs:
  - job: format
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: "format | ./WORKSPACE | **/*.bzl"
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh format'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        AZP_BRANCH: $(Build.SourceBranch)
      displayName: "Run format checks"
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff"
        artifactName: format
      timeoutInMinutes: 10
      # not all have fixes so improve condition/handling
      condition: failed()

  - job: proto_format
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: "proto_format | ./WORKSPACE | **/*.bzl"
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Run check format scripts"

    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/fix_proto_format.diff"
        artifactName: format
      timeoutInMinutes: 10
      condition: failed()

  - job: docs
    dependsOn: []  # this removes the implicit dependency on previous stage and causes this to run in parallel.
    condition: ne(variables['PostSubmit'], true)
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: "docs | ./WORKSPACE | **/*.bzl"
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        AZP_BRANCH: $(Build.SourceBranch)
        AZP_COMMIT_SHA: $(system.pullRequest.sourceCommitId)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Generate docs"

    - script: |
        ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
      displayName: "Upload Docs to GCS"
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)

  - job: dependencies
    dependsOn: []  # this removes the implicit dependency on previous stage and causes this to run in parallel.
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: "dependencies | ./WORKSPACE | **/*.bzl"
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
      displayName: "Verify dependency information"

- stage: sync
  condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
  dependsOn: []
  jobs:
  - job: filter_example
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
        sshPublicKey: "$(FilterExamplePublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(FilterExamplePrivateKey)"

    - bash: ci/filter_example_mirror.sh
      displayName: "Sync envoy-filter-example"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)

  - job: data_plane_api
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
        sshPublicKey: "$(DataPlaneApiPublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(DataPlaneApiPrivateKey)"

    - bash: ci/api_mirror.sh
      displayName: "Sync data-plane-api"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)

  - job: go_control_plane
    dependsOn: []
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
        sshPublicKey: "$(GoControlPlanePublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(GoControlPlanePrivateKey)"

    - bash: |
        cp -a ~/.ssh $(Build.StagingDirectory)/
        ci/run_envoy_docker.sh 'ci/go_mirror.sh'
      displayName: "Sync go-control-plane"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        AZP_BRANCH: $(Build.SourceBranch)

- stage: linux_x64
  dependsOn: ["env", "precheck"]
  # For post-submit builds, continue even if precheck fails
  condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
  jobs:
  - job: release
    condition: and(not(canceled()), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - bash: .azure-pipelines/cleanup.sh
      displayName: "Removing tools from agent"
    - template: bazel.yml
      parameters:
        ciTarget: bazel.release

  - job: released
    dependsOn: ["release"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')
    steps:
    - checkout: none
    - bash: |
        echo "linux_x64 released"

- stage: linux_arm64
  dependsOn: ["env", "precheck"]
  # For post-submit builds, continue even if precheck fails
  condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
  jobs:
  - job: release
    condition: and(not(canceled()), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 120
    pool: "arm-large"
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.release
        rbe: false
        artifactSuffix: ".arm64"
        bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"

  - job: released
    dependsOn: ["release"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')
    steps:
    - checkout: none
    - bash: |
        echo "linux_arm64 released"

- stage: check
  dependsOn: ["env", "linux_x64"]
  jobs:
  - job: bazel
    displayName: "linux_x64"
    dependsOn: []
    # Skip checks if only mobile/ or docs/ have changed.
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    strategy:
      maxParallel: 3
      matrix:
        api:
          CI_TARGET: "bazel.api"
        # Disabled due to https://github.com/envoyproxy/envoy/pull/18218
        # api_compat:
        #  CI_TARGET: "bazel.api_compat"
        gcc:
          CI_TARGET: "bazel.gcc"
        clang_tidy:
          CI_TARGET: "bazel.clang_tidy"
        asan:
          CI_TARGET: "bazel.asan"
        msan:
          CI_TARGET: "bazel.msan"
        tsan:
          CI_TARGET: "bazel.tsan"
        compile_time_options:
          CI_TARGET: "bazel.compile_time_options"
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - template: bazel.yml
      parameters:
        ciTarget: $(CI_TARGET)

  - job: coverage
    displayName: "linux_x64"
    dependsOn: []
    # Skip checks if only mobile/ or docs/ have changed.
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 180
    pool: "x64-large"
    strategy:
      maxParallel: 2
      matrix:
        coverage:
          CI_TARGET: "coverage"
        fuzz_coverage:
          CI_TARGET: "fuzz_coverage"
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.$(CI_TARGET)
        rbe: false
        # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
        bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"

    - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)'
      displayName: "Upload $(CI_TARGET) Report to GCS"
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
      condition: always()

  - job: complete
    displayName: "linux_x64"
    dependsOn: ["bazel", "coverage"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        in(dependencies.bazel.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.coverage.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "checks complete"

- stage: publish
  dependsOn: ["env", "linux_x64", "linux_arm64"]
  jobs:
  - job: docker
    displayName: "Docker (linux multi arch)"
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - bash: |
        echo "disk space at beginning of build:"
        df -h
      displayName: "Check disk space at beginning"
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/envoy-contrib_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - bash: |
        set -e
        mkdir -p linux/amd64
        tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64
        tar zxf $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz -C ./linux/amd64
        rm $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
        rm $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz
        mkdir -p linux/arm64
        tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64
        tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz -C ./linux/arm64
        rm $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz
        rm $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz
        ci/docker_ci.sh
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        AZP_SHA1: $(Build.SourceVersion)
        DOCKERHUB_USERNAME: $(DockerUsername)
        DOCKERHUB_PASSWORD: $(DockerPassword)
    - bash: |
        echo "disk space at end of build:"
        df -h
      displayName: "Check disk space at end"
      condition: always()
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/build_images"
        artifactName: docker
      timeoutInMinutes: 10
      condition: always()

  - job: package_x64
    displayName: Linux debs (x64)
    dependsOn: []
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - template: bazel.yml
      parameters:
        ciTarget: bazel.distribution
  - job: package_arm64
    displayName: Linux debs (arm64)
    dependsOn: []
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool: "arm-large"
    steps:
    - template: bazel.yml
      parameters:
        managedAgent: false
        ciTarget: bazel.distribution
        rbe: false
        artifactSuffix: ".arm64"
        bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"

  - job: success
    dependsOn: ["docker", "package_x64", "package_arm64"]
    displayName: Success (linux artefacts)
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.package_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.package_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "building publishable assets complete"

  - job: docs
    displayName: Publish docs to netlify
    dependsOn: ["success"]
    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: Cache@2
      inputs:
        key: "docs | ./WORKSPACE | **/*.bzl"
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true

    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs)"
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Generate docs"

    - script: |
        ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
      displayName: "Upload Docs to GCS"
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
      condition: eq(variables['isMain'], 'true')

    - task: InstallSSHKey@0
      inputs:
        hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
        sshPublicKey: "$(DocsPublicKey)"
        sshPassphrase: "$(SshDeployKeyPassphrase)"
        sshKeySecureFile: "$(DocsPrivateKey)"

    - script: docs/publish.sh
      displayName: "Publish to GitHub"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        NETLIFY_TRIGGER_URL: $(NetlifyTriggerURL)

  - job: github
    displayName: Publish release tag
    dependsOn: ["docs"]
    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true), eq(variables['isStableBranch'], 'true'))
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - bash: |
        VERSION_DEV="$(cat VERSION.txt | cut -d- -f2)"
        if [[ $VERSION_DEV != "dev" ]]; then
            echo "##vso[task.setvariable variable=isDev]false"
        fi
      displayName: Set development/release env

    - script: ./ci/run_envoy_docker.sh './ci/do_ci.sh publish'
      displayName: "Publish release to Github"
      workingDirectory: $(Build.SourcesDirectory)
      condition: eq(variables['isDev'], 'false')
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)

  - job: assets
    displayName: Publish release assets
    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true), eq(variables['isTaggedRelease'], 'true'))
    dependsOn: ["success"]
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release"
        itemPattern: "bazel.release/envoy-contrib_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.release.arm64"
        itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)

    - task: DownloadSecureFile@1
      name: MaintainerGPGKey
      displayName: 'Download maintainer GPG key'
      inputs:
        # NB: This is the path to the key and **must** be set on a per-pipeline basis
        secureFile: '$(maintainerGPGKey)'

    - bash: |
        set -e

        VERSION="$(cat VERSION.txt)"

        mkdir -p linux/amd64 linux/arm64 publish

        # linux/amd64
        tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64
        tar zxf $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz -C ./linux/amd64
        cp -a linux/amd64/build_envoy_release_stripped/envoy "publish/envoy-${VERSION}-linux-x86_64"
        cp -a linux/amd64/build_envoy-contrib_release_stripped/envoy "publish/envoy-contrib-${VERSION}-linux-x86_64"

        # linux/arm64
        tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64
        tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz -C ./linux/arm64
        cp -a linux/arm64/build_envoy_release_stripped/envoy "publish/envoy-${VERSION}-linux-aarch_64"
        cp -a linux/arm64/build_envoy-contrib_release_stripped/envoy "publish/envoy-contrib-${VERSION}-linux-aarch_64"

        echo "$MAINTAINER_GPG_KEY_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$(MaintainerGPGKey.secureFilePath)"

        ci/publish_github_assets.sh "v${VERSION}" "${PWD}/publish"
      workingDirectory: $(Build.SourcesDirectory)
      env:
        GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
        MAINTAINER_GPG_KEY_PASSPHRASE: $(MaintainerGPGKeyPassphrase)

- stage: verify
  dependsOn: ["env", "publish"]
  jobs:
  - job: examples
    displayName: Verify (Docker/x64) examples
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "docker"
        itemPattern: "docker/envoy*.tar"
        targetPath: $(Build.StagingDirectory)
    - bash: ./ci/do_ci.sh verify_examples
      env:
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        NO_BUILD_SETUP: 1

  - job: packages_x64
    displayName: Verify debs (x64)
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool:
      vmImage: "ubuntu-20.04"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.distribution"
        itemPattern: "bazel.distribution/packages.x64.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Verify packages"
  - job: packages_arm64
    displayName: Verify debs (arm64)
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 120
    pool: "arm-large"
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "bazel.distribution.arm64"
        itemPattern: "bazel.distribution.arm64/packages.arm64.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro'
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
      displayName: "Verify packages"

  - job: verified
    dependsOn: ["examples", "packages_x64", "packages_arm64"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) check passes if all of
    # the preceding checks either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        in(dependencies.examples.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.packages_x64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.packages_arm64.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "checks complete"

- stage: macos
  dependsOn: ["env", "precheck"]
  jobs:
  - job: test
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 180
    pool:
      vmImage: "macos-11"
    steps:
    - script: ./ci/mac_ci_setup.sh
      displayName: "Install dependencies"

    - script: ./ci/mac_ci_steps.sh
      displayName: "Run Mac CI"
      env:
        BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2"
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

    - task: PublishTestResults@2
      inputs:
        testResultsFiles: "**/bazel-testlogs/**/test.xml"
        testRunTitle: "macOS"
      timeoutInMinutes: 10
      condition: always()

    - script: bazel run //ci/flaky_test:process_xml
      displayName: "Process Test Results"
      env:
        TEST_TMPDIR: $(Build.SourcesDirectory)
        SLACK_TOKEN: $(SLACK_TOKEN)
        CI_TARGET: "MacOS"
        REPO_URI: $(Build.Repository.Uri)
        BUILD_URI: $(Build.BuildUri)

  - job: tested
    dependsOn: ["test"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: in(dependencies.test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')
    steps:
    - checkout: none
    - bash: |
        echo "macos tested"

- stage: windows
  dependsOn: ["env", "precheck"]
  jobs:
  - job: release
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    timeoutInMinutes: 180
    pool:
      vmImage: "windows-2019"
    steps:
    - task: Cache@2
      inputs:
        key: '"windows.release" | ./WORKSPACE | **/*.bzl'
        path: $(Build.StagingDirectory)/repository_cache
      continueOnError: true
    - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh
      displayName: "Run Windows msvc-cl CI"
      env:
        CI_TARGET: "windows"
        ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)"
        SLACK_TOKEN: $(SLACK_TOKEN)
        REPO_URI: $(Build.Repository.Uri)
        BUILD_URI: $(Build.BuildUri)
        ENVOY_RBE: "true"
        BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2"
        BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
        BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)

    - task: PublishTestResults@2
      inputs:
        testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
        testRunTitle: "windows"
        searchFolder: $(Build.StagingDirectory)/tmp
      timeoutInMinutes: 10
      condition: always()
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/envoy"
        artifactName: windows.release
      timeoutInMinutes: 10
      condition: always()

  - job: docker
    condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
    strategy:
      matrix:
        windows2019:
          imageName: 'windows-2019'
          windowsBuildType: "windows"
          windowsImageBase: "mcr.microsoft.com/windows/servercore"
          windowsImageTag: "ltsc2019"
        windows2022:
          imageName: 'windows-2022'
          windowsBuildType: "windows-ltsc2022"
          windowsImageBase: "mcr.microsoft.com/windows/nanoserver"
          windowsImageTag: "ltsc2022"
    dependsOn: ["release"]
    timeoutInMinutes: 120
    pool:
      vmImage: $(imageName)
    steps:
    - task: DownloadBuildArtifacts@0
      inputs:
        buildType: current
        artifactName: "windows.release"
        itemPattern: "windows.release/envoy_binary.tar.gz"
        downloadType: single
        targetPath: $(Build.StagingDirectory)
    - bash: |
        set -e
        # Convert to Unix-style path so tar doesn't think drive letter is a hostname
        STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')"
        mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64
        ci/docker_ci.sh
      workingDirectory: $(Build.SourcesDirectory)
      env:
        AZP_BRANCH: $(Build.SourceBranch)
        AZP_SHA1: $(Build.SourceVersion)
        DOCKERHUB_USERNAME: $(DockerUsername)
        DOCKERHUB_PASSWORD: $(DockerPassword)
        WINDOWS_BUILD_TYPE: $(windowsBuildType)
        WINDOWS_IMAGE_BASE: $(windowsImageBase)
        WINDOWS_IMAGE_TAG: $(windowsImageTag)
    - task: PublishBuildArtifacts@1
      inputs:
        pathtoPublish: "$(Build.StagingDirectory)/build_images"
        artifactName: docker_windows
      timeoutInMinutes: 10
      condition: always()

  - job: released
    dependsOn: ["release", "docker"]
    pool:
      vmImage: "ubuntu-20.04"
    # This condition ensures that this (required) job passes if all of
    # the preceeding jobs either pass or are skipped
    # adapted from:
    #   https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage
    condition: |
      and(
        in(dependencies.release.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
        in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
    steps:
    - checkout: none
    - bash: |
        echo "windows released"