Merge attempt to upstream/36cf26b9d6ac32f5671c24b92f9ea22ab70bc6c1

.azure-pipelines/linux.yml

@@ -1,14 +1,62 @@
-resources:
- containers:
- - container: envoy-build
- image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98
-
 jobs:
-- job: BuildEnvoy
+- job: bazel
+ strategy:
+ matrix:
+ release:
+ CI_TARGET: 'bazel.release'
+ tsan:
+ CI_TARGET: 'bazel.tsan'
+ gcc:
+ CI_TARGET: 'bazel.gcc'
+ compile_time_options:
+ CI_TARGET: 'bazel.compile_time_options'
+ fuzz:
+ CI_TARGET: 'bazel.fuzz'
+ dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
  timeoutInMinutes: 360
  pool:
  vmImage: 'Ubuntu 16.04'
- container: envoy-build
  steps:
- - script: bazel build //source/exe:envoy-static
+ - bash: |
+ echo "disk space at beginning of build:"
+ df -h
+ displayName: "Check disk space at beginning"
+
+ - bash: |
+ sudo mkdir -p /etc/docker
+ echo '{
+ "ipv6": true,
+ "fixed-cidr-v6": "2001:db8:1::/64"
+ }' | sudo tee /etc/docker/daemon.json
+ sudo service docker restart
+ displayName: "Enable IPv6"
+
+ - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)'
+ workingDirectory: $(Build.SourcesDirectory)
+ env:
+ ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+ ENVOY_RBE: "true"
+ BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no"
+ BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+ BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+ GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+ displayName: "Run CI script"
+
+ - bash: |
+ echo "disk space at end of build:"
+ df -h
+ displayName: "Check disk space at end"
+ condition: always()
+
+ - task: PublishTestResults@2
+ inputs:
+ testResultsFiles: '**/bazel-out/**/testlogs/**/test.xml'
+ testRunTitle: '$(CI_TARGET)'
+ searchFolder: $(Build.StagingDirectory)/tmp
+ condition: always()
 
+ - task: PublishBuildArtifacts@1
+ inputs:
+ pathtoPublish: "$(Build.StagingDirectory)/envoy"
+ artifactName: $(CI_TARGET)
+ condition: always()

.azure-pipelines/macos.yml

@@ -15,7 +15,7 @@ jobs:
  - script: ./ci/mac_ci_steps.sh
  displayName: 'Run Mac CI'
  env:
- BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com
+ BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
  BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
  GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
 

.bazelci/presubmit.yml

@@ -1,8 +1,11 @@
 ---
 tasks:
- ubuntu_1804:
+ release:
  platform: ubuntu1804
  build_targets:
- - "//source/..."
+ - "//source/exe:envoy-static"
  test_targets:
  - "//test/..."
+ test_flags:
+ # Workaround for https://github.com/envoyproxy/envoy/issues/7647
+ - "--deleted_packages=//test/extensions/tracers/dynamic_ot"

.bazelrc

@@ -1,25 +1,48 @@
 # Envoy specific Bazel build/test options.
 
-# Bazel doesn't need more than 200MB of memory based on memory profiling:
+# Bazel doesn't need more than 200MB of memory for local build based on memory profiling:
 # https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling
+# The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large
+# enough to consume all memory constrained by cgroup in large host, which is the case in CircleCI.
 # Limiting JVM heapsize here to let it do GC more when approaching the limit to
 # leave room for compiler/linker.
-startup --host_jvm_args=-Xmx512m
+# The number 2G is choosed heuristically to both support in CircleCI and large enough for RBE.
+# Startup options cannot be selected via config.
+startup --host_jvm_args=-Xmx2g
+
 build --workspace_status_command=bazel/get_workspace_status
 build --experimental_remap_main_repo
+build --experimental_local_memory_estimate
+build --experimental_strict_action_env=true
+build --host_force_python=PY2
+build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a
+build --action_env=BAZEL_LINKOPTS=-lm
+build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11
+build --javabase=@bazel_tools//tools/jdk:remote_jdk11
+
+# We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace.
+build --define absl=1
+
+# Pass PATH, CC and CXX variables from the environment.
+build --action_env=CC
+build --action_env=CXX
+build --action_env=PATH
+
+# Common flags for sanitizers
+build:sanitizer --define tcmalloc=disabled
+build:sanitizer --linkopt -ldl
+build:sanitizer --build_tag_filters=-no_san
+build:sanitizer --test_tag_filters=-no_san
 
 # Basic ASAN/UBSAN that works for gcc
+build:asan --config=sanitizer
+# ASAN install its signal handler, disable ours so the stacktrace will be printed by ASAN
+build:asan --define signal_trace=disabled
 build:asan --define ENVOY_CONFIG_ASAN=1
 build:asan --copt -fsanitize=address,undefined
 build:asan --linkopt -fsanitize=address,undefined
 build:asan --copt -fno-sanitize=vptr
 build:asan --linkopt -fno-sanitize=vptr
-build:asan --linkopt -fuse-ld=lld
-build:asan --linkopt -ldl
-build:asan --define tcmalloc=disabled
-build:asan --build_tag_filters=-no_asan
-build:asan --test_tag_filters=-no_asan
-build:asan --define signal_trace=disabled
 build:asan --copt -DADDRESS_SANITIZER=1
 build:asan --copt -D__SANITIZE_ADDRESS__
 build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1
@@ -28,41 +51,104 @@ build:asan --test_env=ASAN_SYMBOLIZER_PATH
 
 # Clang ASAN/UBSAN
 build:clang-asan --config=asan
+build:clang-asan --linkopt -fuse-ld=lld
 
 # macOS ASAN/UBSAN
 build:macos-asan --config=asan
 # Workaround, see https://github.com/bazelbuild/bazel/issues/6932
 build:macos-asan --copt -Wno-macro-redefined
 build:macos-asan --copt -D_FORTIFY_SOURCE=0
+# Workaround, see https://github.com/bazelbuild/bazel/issues/4341
+build:macos-asan --copt -DGRPC_BAZEL_BUILD
+# Dynamic link cause issues like: `dyld: malformed mach-o: load commands size (59272) > 32768`
+build:macos-asan --dynamic_mode=off
 
 # Clang TSAN
+build:clang-tsan --config=sanitizer
 build:clang-tsan --define ENVOY_CONFIG_TSAN=1
 build:clang-tsan --copt -fsanitize=thread
 build:clang-tsan --linkopt -fsanitize=thread
 build:clang-tsan --linkopt -fuse-ld=lld
-build:clang-tsan --define tcmalloc=disabled
 # Needed due to https://github.com/libevent/libevent/issues/777
 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE
 
 # Clang MSAN - broken today since we need to rebuild lib[std]c++ and external deps with MSAN
 # support (see https://github.com/envoyproxy/envoy/issues/443).
+build:clang-msan --config=sanitizer
 build:clang-msan --define ENVOY_CONFIG_MSAN=1
 build:clang-msan --copt -fsanitize=memory
 build:clang-msan --linkopt -fsanitize=memory
-build:clang-msan --define tcmalloc=disabled
 build:clang-msan --copt -fsanitize-memory-track-origins=2
 
 # Clang with libc++
 # TODO(cmluciano) fix and re-enable _LIBCPP_VERSION testing for TCMALLOC in Envoy::Stats::TestUtil::hasDeterministicMallocStats
 # and update stats_integration_test with appropriate m_per_cluster value
-build:libc++ --action_env=CC
-build:libc++ --action_env=CXX
 build:libc++ --action_env=CXXFLAGS=-stdlib=libc++
-build:libc++ --action_env=PATH
+build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++
+build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm
+build:libc++ --host_linkopt=-fuse-ld=lld
 build:libc++ --define force_libcpp=enabled
 
 # Optimize build for binary size reduction.
 build:sizeopt -c opt --copt -Os
 
 # Test options
-test --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
+build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
+
+# Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
+build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+
+build:rbe-toolchain-clang --config=rbe-toolchain
+build:rbe-toolchain-clang --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:rbe-toolchain-clang --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:rbe-toolchain-clang --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:rbe-toolchain-clang-libc++ --config=rbe-toolchain
+build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain
+build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain
+build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled
+
+build:rbe-toolchain-gcc --config=rbe-toolchain
+build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain
+build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain
+
+build:remote --spawn_strategy=remote,sandboxed,local
+build:remote --strategy=Javac=remote,sandboxed,local
+build:remote --strategy=Closure=remote,sandboxed,local
+build:remote --strategy=Genrule=remote,sandboxed,local
+build:remote --remote_timeout=3600
+build:remote --auth_enabled=true
+build:remote --experimental_inmemory_jdeps_files
+build:remote --experimental_inmemory_dotd_files
+build:remote --experimental_remote_download_outputs=toplevel
+
+build:remote-clang --config=remote
+build:remote-clang --config=rbe-toolchain-clang
+
+# Docker sandbox
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f
+build:docker-sandbox --spawn_strategy=docker
+build:docker-sandbox --strategy=Javac=docker
+build:docker-sandbox --strategy=Closure=docker
+build:docker-sandbox --strategy=Genrule=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+
+build:docker-clang --config=docker-sandbox
+build:docker-clang --config=rbe-toolchain-clang
+
+# CI configurations
+build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
+build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Fuzz builds
+build:asan-fuzzer --config=asan
+build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer
+build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard
+# Remove UBSAN halt_on_error to avoid crashing on protobuf errors.
+build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1

.bazelversion

@@ -0,0 +1 @@
+0.29.1

.circleci/config.yml

@@ -4,7 +4,8 @@ executors:
  ubuntu-build:
  description: "A regular build executor based on ubuntu image"
  docker:
- - image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98
+ # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here.
+ - image: envoyproxy/envoy-build:cb15cc3d2010aff77d6e022ddf6d723fa8becbc0
  resource_class: xlarge
  working_directory: /source
 
@@ -52,25 +53,7 @@ jobs:
  - store_artifacts:
  path: /build/envoy/generated
  destination: /
- tsan:
- executor: ubuntu-build
- steps:
- - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
- - checkout
- - run: ci/do_circle_ci.sh bazel.tsan
- - store_artifacts:
- path: /build/envoy/generated
- destination: /
 
- compile_time_options:
- executor: ubuntu-build
- steps:
- - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
- - checkout
- - run: ci/do_circle_ci.sh bazel.compile_time_options
- - store_artifacts:
- path: /build/envoy/generated
- destination: /
  api:
  executor: ubuntu-build
  steps:
@@ -93,28 +76,6 @@ jobs:
  fingerprints:
  - "f6:f9:df:90:9c:4b:5f:9c:f4:69:fd:42:94:ff:88:24"
  - run: ci/filter_example_mirror.sh
- ipv6_tests:
- machine: true
- steps:
- - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
- - checkout
- - run:
- name: enable ipv6
- command: |
- echo '{
- "ipv6": true,
- "fixed-cidr-v6": "2001:db8:1::/64"
- }' | sudo tee /etc/docker/daemon.json
-
- sudo service docker restart
- - run: dig go.googlesource.com A go.googlesource.com AAAA # Debug IPv6 network issues
- - run: ifconfig
- - run: route -A inet -A inet6
- - run: curl -v https://go.googlesource.com
- - run: curl -6 -v https://go.googlesource.com || true
- - run: ./ci/do_circle_ci_ipv6_tests.sh
- - store_artifacts:
- path: /tmp/envoy-docker/envoy/generated/failed-testlogs
 
  coverage:
  executor: ubuntu-build
@@ -125,11 +86,24 @@ jobs:
  command:
  ci/do_circle_ci.sh bazel.coverage
  no_output_timeout: 60m
- - run: ci/coverage_publish.sh
+ - persist_to_workspace:
+ root: /build/envoy/generated
+ paths:
+ - coverage
  - store_artifacts:
  path: /build/envoy/generated
  destination: /
 
+ coverage_publish:
+ docker:
+ - image: google/cloud-sdk
+ steps:
+ - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
+ - checkout
+ - attach_workspace:
+ at: /build/envoy/generated
+ - run: ci/coverage_publish.sh
+
  clang_tidy:
  executor: ubuntu-build
  steps:
@@ -153,11 +127,7 @@ jobs:
  - run: ci/do_circle_ci.sh check_spelling_pedantic
  build_image:
  docker:
- - image: circleci/python:3.7
- environment:
- # See comment in do_circle_ci.sh for why we do this.
- NUM_CPUS: 8
- resource_class: xlarge
+ - image: google/cloud-sdk
  steps:
  - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
  - checkout
@@ -190,12 +160,11 @@ workflows:
  tags:
  only: /^v.*/
  - asan
- - tsan
- - compile_time_options
  - api
  - filter_example_mirror
- - ipv6_tests
  - coverage
+ - coverage_publish:
+ requires: [coverage]
  - format
  - clang_tidy
  - build_image