sync from master.

.azure-pipelines/bazel.yml

@@ -66,6 +66,8 @@ steps:
   - bash: |
       echo "disk space at end of build:"
       df -h
+      # Cleanup offending files with unicode names
+      rm -rf $(Build.StagingDirectory)/tmp/*/*/external/go_sdk/test/fixedbugs
     displayName: "Check disk space at end"
     condition: always()
 

.azure-pipelines/cleanup.sh

@@ -3,7 +3,7 @@
 set -e
 
 # Temporary script to remove tools from Azure pipelines agent to create more disk space room.
-sudo apt-get update -y
+sudo apt-get update -y || true
 sudo apt-get purge -y --no-upgrade 'ghc-*' 'zulu-*-azure-jdk' 'libllvm*' 'mysql-*' 'dotnet-*' 'libgl1' \
   'adoptopenjdk-*' 'azure-cli' 'google-chrome-stable' 'firefox' 'hhvm'
 

.azure-pipelines/cve_scan.yml

@@ -0,0 +1,22 @@
+# Pipeline for running Envoy's CVE scanner on an hourly basis.
+
+# This pipeline only uses scheduled triggers.
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 * * * *"
+  displayName: Hourly CVE scan
+  branches:
+    include:
+    - master
+  always: true
+
+pool:
+  vmImage: "ubuntu-18.04"
+steps:
+  - script: ci/run_envoy_docker.sh 'ci/do_ci.sh cve_scan'
+    workingDirectory: $(Build.SourcesDirectory)
+    env:
+      ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+    displayName: "Scan for CVEs in dependencies"

.azure-pipelines/pipelines.yml

@@ -41,6 +41,7 @@ stages:
 
       - job: docs
         dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        condition: ne(variables['PostSubmit'], true)
         pool:
           vmImage: "ubuntu-18.04"
         steps:
@@ -59,37 +60,31 @@ stages:
               GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
             displayName: "Generate docs"
 
-          - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
+          - script: |
+              ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
             displayName: "Upload Docs to GCS"
             env:
               ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
               GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
               GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
 
-          - task: PublishBuildArtifacts@1
-            inputs:
-              pathtoPublish: "$(Build.SourcesDirectory)/generated/docs"
-              artifactName: docs
-            condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
-
-          - task: InstallSSHKey@0
-            inputs:
-              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
-              sshPublicKey: "$(DocsPublicKey)"
-              sshPassphrase: "$(SshDeployKeyPassphrase)"
-              sshKeySecureFile: "$(DocsPrivateKey)"
-            condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
-
-          - script: docs/publish.sh
-            displayName: "Publish to GitHub"
+      - job: dependencies
+        dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        pool:
+          vmImage: "ubuntu-18.04"
+        steps:
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps'
             workingDirectory: $(Build.SourcesDirectory)
             env:
-              AZP_BRANCH: $(Build.SourceBranch)
-              AZP_SHA1: $(Build.SourceVersion)
-            condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['PostSubmit'], true))
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken)
+            displayName: "Verify dependency information"
 
   - stage: sync
-    condition: and(succeeded(), eq(variables['PostSubmit'], true))
+    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
     dependsOn: []
     jobs:
       - job: filter_example
@@ -130,6 +125,8 @@ stages:
 
       - job: go_control_plane
         dependsOn: []
+        pool:
+          vmImage: "ubuntu-18.04"
         steps:
           - task: InstallSSHKey@0
             inputs:
@@ -152,12 +149,11 @@ stages:
 
   - stage: linux_x64
     dependsOn: ["precheck"]
-    # For master builds, continue even if precheck fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
     jobs:
       - job: release
-        # For master builds, continue even if format fails
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "ubuntu-18.04"
         steps:
@@ -167,11 +163,11 @@ stages:
 
   - stage: linux_arm64
     dependsOn: ["precheck"]
-    # For master builds, continue even if precheck fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
     jobs:
       - job: release
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool: "arm-large"
         steps:
           - template: bazel.yml
@@ -203,7 +199,7 @@ stages:
               CI_TARGET: "bazel.tsan"
             compile_time_options:
               CI_TARGET: "bazel.compile_time_options"
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "ubuntu-18.04"
         steps:
@@ -214,7 +210,7 @@ stages:
       - job: coverage
         displayName: "linux_x64"
         dependsOn: []
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool: "x64-large"
         strategy:
           maxParallel: 2
@@ -290,6 +286,43 @@ stages:
               artifactName: docker
             condition: always()
 
+  - stage: docs
+    dependsOn: ["docker"]
+    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
+    jobs:
+      - job: publish
+        pool:
+          vmImage: "ubuntu-18.04"
+        steps:
+          - task: Cache@2
+            inputs:
+              key: "docs | ./WORKSPACE | **/*.bzl"
+              path: $(Build.StagingDirectory)/repository_cache
+            continueOnError: true
+
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+            displayName: "Generate docs"
+
+          - task: InstallSSHKey@0
+            inputs:
+              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+              sshPublicKey: "$(DocsPublicKey)"
+              sshPassphrase: "$(SshDeployKeyPassphrase)"
+              sshKeySecureFile: "$(DocsPrivateKey)"
+
+          - script: docs/publish.sh
+            displayName: "Publish to GitHub"
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              AZP_BRANCH: $(Build.SourceBranch)
+              AZP_SHA1: $(Build.SourceVersion)
+
   - stage: verify
     dependsOn: ["docker"]
     jobs:
@@ -313,7 +346,7 @@ stages:
     dependsOn: ["precheck"]
     jobs:
       - job: test
-        timeoutInMinutes: 360
+        timeoutInMinutes: 180
         pool:
           vmImage: "macos-latest"
         steps:
@@ -347,19 +380,29 @@ stages:
     dependsOn: ["precheck"]
     jobs:
       - job: release
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "windows-latest"
         steps:
           - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh
             displayName: "Run Windows CI"
             env:
+              CI_TARGET: "windows"
               ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)"
+              SLACK_TOKEN: $(SLACK_TOKEN)
+              REPO_URI: $(Build.Repository.Uri)
+              BUILD_URI: $(Build.BuildUri)
               ENVOY_RBE: "true"
-              BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)"
+              BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2"
               BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
               BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
               GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+          - task: PublishTestResults@2
+            inputs:
+              testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml"
+              testRunTitle: "windows"
+              searchFolder: $(Build.StagingDirectory)/tmp
+            condition: always()
           - task: PublishBuildArtifacts@1
             inputs:
               pathtoPublish: "$(Build.StagingDirectory)/envoy"
@@ -368,7 +411,7 @@ stages:
 
       - job: docker
         dependsOn: ["release"]
-        timeoutInMinutes: 360
+        timeoutInMinutes: 120
         pool:
           vmImage: "windows-latest"
         steps:

.bazelci/presubmit.yml

@@ -10,6 +10,7 @@ tasks:
     test_flags:
       - "--config=remote-clang-libc++"
       - "--config=remote-ci"
+      - "--define=wasm=disabled"
       - "--jobs=75"
   coverage:
     name: "Coverage"
@@ -23,3 +24,14 @@ tasks:
     test_flags:
       - "--config=coverage"
       - "--config=clang"
+  fuzz_coverage:
+    name: "Fuzz-Coverage"
+    platform: ubuntu1804
+    shell_commands:
+      - "bazel/setup_clang.sh /usr/lib/llvm-10"
+    test_targets:
+      - "//test/server:server_fuzz_test"
+    test_flags:
+      - "--config=fuzz-coverage"
+      - "--config=coverage"
+      - "--config=clang"

.bazelrc

@@ -62,6 +62,9 @@ build:asan --copt -D__SANITIZE_ADDRESS__
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1
 build:asan --test_env=UBSAN_OPTIONS=halt_on_error=true:print_stacktrace=1
 build:asan --test_env=ASAN_SYMBOLIZER_PATH
+# ASAN needs -O1 to get reasonable performance.
+build:asan --copt -O1
+build:asan --copt -fno-optimize-sibling-calls
 
 # Clang ASAN/UBSAN
 build:clang-asan --config=asan
@@ -141,9 +144,9 @@ build:coverage --experimental_use_llvm_covmap
 build:coverage --collect_code_coverage
 build:coverage --test_tag_filters=-nocoverage
 build:coverage --instrumentation_filter="//source(?!/common/chromium_url|/extensions/quic_listeners/quiche/platform)[/:],//include[/:]"
-coverage:test-coverage --test_arg="-l trace"
-coverage:fuzz-coverage --config=plain-fuzzer
-coverage:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
+build:test-coverage --test_arg="-l trace"
+build:fuzz-coverage --config=plain-fuzzer
+build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh
 
 # Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
 build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
@@ -165,6 +168,16 @@ build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++
 build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++
 build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled
 
+# Do not inherit from "clang-asan" to avoid picking up flags from local clang.bazelrc.
+build:rbe-toolchain-asan --config=asan
+build:rbe-toolchain-asan --linkopt -fuse-ld=lld
+build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1
+build:rbe-toolchain-asan --copt=-fsanitize=vptr,function
+build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function
+build:rbe-toolchain-asan --linkopt=-L/opt/llvm/lib/clang/10.0.0/lib/linux
+build:rbe-toolchain-asan --linkopt=-l:libclang_rt.ubsan_standalone-x86_64.a
+build:rbe-toolchain-asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx-x86_64.a
+
 build:rbe-toolchain-msan --linkopt=-L/opt/libcxx_msan/lib
 build:rbe-toolchain-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
 build:rbe-toolchain-msan --config=clang-msan
@@ -215,6 +228,10 @@ build:remote-clang-libc++ --config=rbe-toolchain-clang-libc++
 build:remote-gcc --config=remote
 build:remote-gcc --config=rbe-toolchain-gcc
 
+build:remote-asan --config=remote
+build:remote-asan --config=rbe-toolchain-clang-libc++
+build:remote-asan --config=rbe-toolchain-asan
+
 build:remote-msan --config=remote
 build:remote-msan --config=rbe-toolchain-clang-libc++
 build:remote-msan --config=rbe-toolchain-msan
@@ -229,7 +246,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl
 
 # Docker sandbox
 # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8
-build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:b480535e8423b5fd7c102fd30c92f4785519e33a
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:11efa5680d987fff33fde4af3cc5ece105015d04
 build:docker-sandbox --spawn_strategy=docker
 build:docker-sandbox --strategy=Javac=docker
 build:docker-sandbox --strategy=Closure=docker
@@ -260,22 +277,18 @@ build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
 
 # Fuzz builds
-# -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION is passed in in the bazel build target
-# rules for fuzz tests. Passing it in the CLI will cause dependencies to be build
-# with the macro. Causing issues in RouteMatcherTest.TestRoutes that expect prod
-# behavior from RE2 library.
-build:asan-fuzzer --config=asan
-build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
-build:asan-fuzzer --copt=-fsanitize=fuzzer-no-link
-build:asan-fuzzer --copt=-fno-omit-frame-pointer
-# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
-build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
-
 # Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts.
 build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer
 build:plain-fuzzer --define ENVOY_CONFIG_ASAN=1
 build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link
 build:plain-fuzzer --linkopt=-fsanitize=fuzzer-no-link
+build:plain-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+
+build:asan-fuzzer --config=plain-fuzzer
+build:asan-fuzzer --config=asan
+build:asan-fuzzer --copt=-fno-omit-frame-pointer
+# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
+build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1
 
 # Compile database generation config
 build:compdb --build_tag_filters=-nocompdb
@@ -285,6 +298,7 @@ build:windows --action_env=TMPDIR
 build:windows --define signal_trace=disabled
 build:windows --define hot_restart=disabled
 build:windows --define tcmalloc=disabled
+build:windows --define wasm=disabled
 build:windows --define manual_stamp=manual_stamp
 build:windows --cxxopt="/std:c++17"
 
@@ -309,6 +323,12 @@ build:clang-cl --define clang_cl=1
 # Override determinism flags (DATE etc) is valid on clang-cl compiler
 build:clang-cl --copt="-Wno-macro-redefined"
 build:clang-cl --copt="-Wno-builtin-macro-redefined"
+# Workaround problematic missing override declarations of mocks
+# TODO: resolve this class of problematic mocks, e.g.
+# ./test/mocks/http/stream.h(16,21): error: 'addCallbacks'
+#     overrides a member function but is not marked 'override'
+#   MOCK_METHOD(void, addCallbacks, (StreamCallbacks & callbacks));
+build:clang-cl --copt="-Wno-inconsistent-missing-override"
 build:clang-cl --action_env=USE_CLANG_CL=1
 
 # Defaults to 'auto' - Off for windows, so override to linux behavior

.bazelversion

@@ -1 +1 @@
-3.4.1
+3.7.2

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/envoy-ci/envoy-build:b480535e8423b5fd7c102fd30c92f4785519e33a
+FROM gcr.io/envoy-ci/envoy-build:11efa5680d987fff33fde4af3cc5ece105015d04
 
 ARG USERNAME=vscode
 ARG USER_UID=501