Merge remote-tracking branch 'upstream/main' into oid-lua-api

* upstream/main: (21 commits)
  Add a CPU utilization resource monitor for overload manager (envoyproxy#34713)
  jwks: Add UA string to headers (envoyproxy#35977)
  exceptions: cleaning up macros (envoyproxy#35694)
  coverage: ratcheting (envoyproxy#36058)
  runtime: load rtds bool correctly as true/false instead of 1/0 (envoyproxy#36044)
  Typo in documentation of http original_src filter (envoyproxy#36060)
  docs: updating meeting info (envoyproxy#36052)
  quic: removes more references to spdy::Http2HeaderBlock. (envoyproxy#36057)
  json: add null support to the streamer (envoyproxy#36051)
  json: make the streamer a template class (envoyproxy#36001)
  docs: Add `apt.envoyproxy.io` install information (envoyproxy#36050)
  ext_proc: elide redundant copy in ext_proc filter factory callback (envoyproxy#36015)
  build(deps): bump yarl from 1.11.0 to 1.11.1 in /tools/base (envoyproxy#36049)
  build(deps): bump multidict from 6.0.5 to 6.1.0 in /tools/base (envoyproxy#36048)
  quic: enable certificate compression/decompression (envoyproxy#35999)
  Geoip fix asan failure (envoyproxy#36043)
  mobile: Fix missing logging output in Swift integration tests (envoyproxy#36040)
  http: minor code clean up to the http filter manager (envoyproxy#36027)
  ci/example: Dont build/test the filter example in Envoy CI (envoyproxy#36038)
  ci/codeql: Fix build setup (envoyproxy#36021)
  ...

Signed-off-by: Qiu Yu <qiuyu@apple.com>

.azure-pipelines/ci.yml

@@ -64,9 +64,6 @@ parameters:
   displayName: "Upload bazel run data to BES"
   type: boolean
   default: true
-- name: envoyBuildFilterExample
-  type: string
-  default: ""
 - name: cacheTestResults
   displayName: "Cache test results"
   type: boolean
@@ -224,7 +221,6 @@ steps:
     echo "##vso[task.setvariable variable=BAZEL_FAKE_SCM_REVISION]${BAZEL_FAKE_SCM_REVISION}"
     echo "##vso[task.setvariable variable=BAZEL_STARTUP_EXTRA_OPTIONS]${{ parameters.bazelStartupExtraOptions }}"
     echo "##vso[task.setvariable variable=CI_TARGET_BRANCH]${CI_TARGET_BRANCH}"
-    echo "##vso[task.setvariable variable=ENVOY_BUILD_FILTER_EXAMPLE]${{ parameters.envoyBuildFilterExample }}"
     echo "##vso[task.setvariable variable=ENVOY_DOCKER_BUILD_DIR]$(Build.StagingDirectory)"
     echo "##vso[task.setvariable variable=ENVOY_RBE]${ENVOY_RBE}"
     echo "##vso[task.setvariable variable=ENVOY_SHARED_TMP_DIR]${ENVOY_SHARED_TMP_DIR}"
@@ -275,7 +271,6 @@ steps:
 - script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
   workingDirectory: $(Build.SourcesDirectory)
   env:
-    ENVOY_BUILD_FILTER_EXAMPLE: ${{ parameters.envoyBuildFilterExample }}
     ${{ each var in parameters.env }}:
       ${{ var.key }}: ${{ var.value }}
   displayName: "Run CI script ${{ parameters.ciTarget }}"

.azure-pipelines/stage/checks.yml

@@ -47,20 +47,17 @@ jobs:
         CI_TARGET: "fuzz_coverage"
       compile_time_options:
         CI_TARGET: "compile_time_options"
-        ENVOY_FILTER_EXAMPLE: true
       tsan:
         CI_TARGET: "tsan"
       asan:
         CI_TARGET: "asan"
-        ENVOY_FILTER_EXAMPLE: true
       # Disabled due to https://github.com/envoyproxy/envoy/pull/18218
       # api_compat:
       #  CI_TARGET: "api_compat"
       gcc:
         CI_TARGET: "gcc"
       msan:
         CI_TARGET: "msan"
-        ENVOY_FILTER_EXAMPLE: true
       #
       # Temporarily disabled to facilitate release CI, should be resolved
       #    as part of https://github.com/envoyproxy/envoy/issues/28566
@@ -81,7 +78,6 @@ jobs:
     parameters:
       ciTarget: $(CI_TARGET)
       cacheName: $(CI_TARGET)
-      envoyBuildFilterExample: $(ENVOY_FILTER_EXAMPLE)
       cacheTestResults: ${{ parameters.cacheTestResults }}
       diskspaceHack: true
       managedAgent: false

.bazelrc

@@ -272,8 +272,7 @@ build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++
 build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++
 build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled
 
-# Do not inherit from "clang-asan" to avoid picking up flags from local clang.bazelrc.
-build:rbe-toolchain-asan --config=asan
+build:rbe-toolchain-asan --config=clang-asan
 build:rbe-toolchain-asan --linkopt -fuse-ld=lld
 build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1
 build:rbe-toolchain-asan --copt=-fsanitize=vptr,function

.github/workflows/codeql-daily.yml

@@ -54,13 +54,18 @@ jobs:
        cd bin/clang14
        wget https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
        tar -xf clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz --strip-components 1
-       export PATH=bin/clang14/bin:$PATH
 
     - name: Build
       run: |
        bazel/setup_clang.sh bin/clang14
        bazelisk shutdown
-       bazelisk build -c fastbuild --spawn_strategy=local --discard_analysis_cache --nouse_action_cache --config clang --config libc++ \
+       bazel build \
+           -c fastbuild \
+           --spawn_strategy=local \
+           --discard_analysis_cache \
+           --nouse_action_cache \
+           --config=clang-libc++ \
+           --config=ci \
            //source/common/http/...
 
     - name: Clean Artifacts

.github/workflows/codeql-push.yml

@@ -85,7 +85,6 @@ jobs:
        cd bin/clang14
        wget https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz
        tar -xf clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz --strip-components 1
-       export PATH=bin/clang14/bin:$PATH
 
     - name: Build
       if: ${{ env.BUILD_TARGETS != '' }}
@@ -97,8 +96,8 @@ jobs:
            --spawn_strategy=local \
            --discard_analysis_cache \
            --nouse_action_cache \
-           --config clang \
-           --config libc++ \
+           --config=clang-libc++ \
+           --config=ci \
            $BUILD_TARGETS
        echo -e "Built targets...\n$BUILD_TARGETS"
 

CODEOWNERS

@@ -142,6 +142,7 @@ extensions/filters/common/original_src @klarose @mattklein123
 /*/extensions/resource_monitors/common @eziskind @yanavlasov @nezdolik
 /*/extensions/resource_monitors/fixed_heap @eziskind @yanavlasov @nezdolik
 /*/extensions/resource_monitors/downstream_connections @nezdolik @mattklein123
+/*/extensions/resource_monitors/cpu_utilization @cancecen @kbaichoo
 /*/extensions/retry/priority @alyssawilk @mattklein123
 /*/extensions/retry/priority/previous_priorities @alyssawilk @mattklein123
 /*/extensions/retry/host @alyssawilk @mattklein123

README.md

@@ -69,11 +69,13 @@ have prior experience. To get started:
 
 ## Community Meeting
 
-The Envoy team meets twice per month on Tuesday at 9am PT. The public
-Google calendar is here: https://goo.gl/PkDijT
-
-* Meeting minutes are [here](https://goo.gl/5Cergb)
-* Recorded videos are posted [here](https://www.youtube.com/channel/UC5z5mvPgqMs1xo5VuIWzYTA)
+The Envoy team has a scheduled meeting time twice per month on Tuesday at 9am PT. The public
+Google calendar is [here](https://goo.gl/PkDijT).  The meeting will only be held
+if there are agenda items listed in the [meeting
+minutes](https://goo.gl/5Cergb).  Any member of the community should be able to
+propose agenda items by adding to the minutes.  The maintainers will either confirm
+the additions to the agenda, or will cancel the meeting within 24 hours of the scheduled
+date if there is no confirmed agenda.
 
 ## Security
 

api/BUILD

@@ -311,6 +311,7 @@ proto_library(
         "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg",
         "//envoy/extensions/regex_engines/v3:pkg",
         "//envoy/extensions/request_id/uuid/v3:pkg",
+        "//envoy/extensions/resource_monitors/cpu_utilization/v3:pkg",
         "//envoy/extensions/resource_monitors/downstream_connections/v3:pkg",
         "//envoy/extensions/resource_monitors/fixed_heap/v3:pkg",
         "//envoy/extensions/resource_monitors/injected_resource/v3:pkg",

api/envoy/extensions/resource_monitors/cpu_utilization/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
+)

api/envoy/extensions/resource_monitors/cpu_utilization/v3/cpu_utilization.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package envoy.extensions.resource_monitors.cpu_utilization.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.resource_monitors.cpu_utilization.v3";
+option java_outer_classname = "CpuUtilizationProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/resource_monitors/cpu_utilization/v3;cpu_utilizationv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: CPU utilization]
+// [#extension: envoy.resource_monitors.cpu_utilization]
+
+// The CPU utilization resource monitor reports the Envoy process the CPU Utilization of the entire host.
+// Today, this only works on Linux and is calculated using the stats in the /proc/stat file.
+message CpuUtilizationConfig {
+}

api/versioning/BUILD

@@ -250,6 +250,7 @@ proto_library(
         "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg",
         "//envoy/extensions/regex_engines/v3:pkg",
         "//envoy/extensions/request_id/uuid/v3:pkg",
+        "//envoy/extensions/resource_monitors/cpu_utilization/v3:pkg",
         "//envoy/extensions/resource_monitors/downstream_connections/v3:pkg",
         "//envoy/extensions/resource_monitors/fixed_heap/v3:pkg",
         "//envoy/extensions/resource_monitors/injected_resource/v3:pkg",

bazel/setup_clang.sh

@@ -15,11 +15,14 @@ fi
 LLVM_VERSION="$("${LLVM_CONFIG}" --version)"
 LLVM_LIBDIR="$("${LLVM_CONFIG}" --libdir)"
 LLVM_TARGET="$("${LLVM_CONFIG}" --host-target)"
+PATH="$("${LLVM_CONFIG}" --bindir):${PATH}"
 
 RT_LIBRARY_PATH="${LLVM_LIBDIR}/clang/${LLVM_VERSION}/lib/${LLVM_TARGET}"
 
 cat <<EOF > "${BAZELRC_FILE}"
 # Generated file, do not edit. If you want to disable clang, just delete this file.
+build:clang --host_action_env=PATH=${PATH} --action_env=PATH=${PATH}
+
 build:clang --action_env=LLVM_CONFIG=${LLVM_CONFIG} --host_action_env=LLVM_CONFIG=${LLVM_CONFIG}
 build:clang --repo_env=LLVM_CONFIG=${LLVM_CONFIG}
 build:clang --linkopt=-L${LLVM_LIBDIR}

changelogs/current.yaml

@@ -128,6 +128,11 @@ bug_fixes:
     the number of requests per I/O cycle is configured and an HTTP decoder filter that pauses filter chain is present. This behavior
     can be reverted by setting the runtime guard ``envoy.reloadable_features.use_filter_manager_state_for_downstream_end_stream``
     to false.
+- area: runtime
+  change: |
+    Fixed an inconsistency in how boolean values are loaded in RTDS, where they were previously converted to "1"/"0"
+    instead of "true"/"false". The correct string representation ("true"/"false") will now be used. This change can be
+    reverted by setting the runtime guard ``envoy.reloadable_features.boolean_to_string_fix`` to false.
 
 removed_config_or_runtime:
 # *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
@@ -230,6 +235,11 @@ new_features:
     Added socket ``type`` field for specifying a socket type to apply the socket option to under :ref:`SocketOption
     <envoy_v3_api_msg_config.core.v3.SocketOption>`. If not specified, the socket option will be applied to all socket
     types.
+- area: quic
+  change: |
+    QUIC server and client support certificate compression, which can in some cases reduce the number of round trips
+    required to setup a connection. This change temporarily disabled by setting the runtime flag
+    ``envoy.reloadable_features.quic_support_certificate_compression`` to ``false``.
 - area: tls
   change: |
     Added an extension point :ref:`custom_tls_certificate_selector
@@ -307,6 +317,10 @@ new_features:
   change: |
     The :ref:`xff <envoy_v3_api_msg_extensions.http.original_ip_detection.xff.v3.XffConfig>`
     original IP detection method now supports using a list of trusted CIDRs when parsing ``x-forwarded-for``.
+- area: resource_monitors
+  change: |
+    Added possibility to monitor CPU utilization in Linux based systems via :ref:`cpu utilization monitor
+    <envoy_v3_api_msg_extensions.resource_monitors.cpu_utilization.v3.CpuUtilizationConfig>` in overload manager.
 - area: lua
   change: |
       Added two new methods ``oidsPeerCertificate()`` and ``oidsLocalCertificate()`` to SSL

ci/build_setup.sh

@@ -33,8 +33,6 @@ fi
     export ENVOY_BUILD_ARCH
 }
 
-export ENVOY_BUILD_FILTER_EXAMPLE="${ENVOY_BUILD_FILTER_EXAMPLE:-0}"
-
 read -ra BAZEL_BUILD_EXTRA_OPTIONS <<< "${BAZEL_BUILD_EXTRA_OPTIONS:-}"
 read -ra BAZEL_EXTRA_TEST_OPTIONS <<< "${BAZEL_EXTRA_TEST_OPTIONS:-}"
 read -ra BAZEL_STARTUP_EXTRA_OPTIONS <<< "${BAZEL_STARTUP_EXTRA_OPTIONS:-}"
@@ -119,10 +117,12 @@ export BAZEL_STARTUP_OPTION_LIST
 export BAZEL_BUILD_OPTION_LIST
 export BAZEL_GLOBAL_OPTION_LIST
 
-if [[ -e "${LLVM_ROOT}" ]]; then
-    "${CURRENT_SCRIPT_DIR}/../bazel/setup_clang.sh" "${LLVM_ROOT}"
-else
-    echo "LLVM_ROOT not found, not setting up llvm."
+if [[ -z "${ENVOY_RBE}" ]]; then
+    if [[ -e "${LLVM_ROOT}" ]]; then
+        "${CURRENT_SCRIPT_DIR}/../bazel/setup_clang.sh" "${LLVM_ROOT}"
+    else
+        echo "LLVM_ROOT not found, not setting up llvm."
+    fi
 fi
 
 [[ "${BAZEL_EXPUNGE}" == "1" ]] && bazel clean "${BAZEL_BUILD_OPTIONS[@]}" --expunge
@@ -155,11 +155,4 @@ mkdir -p "${ENVOY_FAILED_TEST_LOGS}"
 export ENVOY_BUILD_PROFILE="${ENVOY_BUILD_DIR}"/generated/build-profile
 mkdir -p "${ENVOY_BUILD_PROFILE}"
 
-if [[ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "true" ]]; then
-  # shellcheck source=ci/filter_example_setup.sh
-  . "${CURRENT_SCRIPT_DIR}"/filter_example_setup.sh
-else
-  echo "Skip setting up Envoy Filter Example."
-fi
-
 export NO_BUILD_SETUP=1

ci/do_ci.sh

@@ -351,22 +351,19 @@ case $CI_TARGET in
 
     asan)
         setup_clang_toolchain
+        if [[ -n "$ENVOY_RBE" ]]; then
+            ASAN_CONFIG="--config=rbe-toolchain-asan"
+        else
+            ASAN_CONFIG="--config=clang-asan"
+        fi
         BAZEL_BUILD_OPTIONS+=(
             -c dbg
-            "--config=clang-asan"
+            "${ASAN_CONFIG}"
             "--build_tests_only"
             "--remote_download_minimal")
         echo "bazel ASAN/UBSAN debug build with tests"
         echo "Building and testing envoy tests ${TEST_TARGETS[*]}"
         bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" "${TEST_TARGETS[@]}"
-        if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then
-            echo "Building and testing envoy-filter-example tests..."
-            pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"
-            bazel_with_collection \
-                test "${BAZEL_BUILD_OPTIONS[@]}" \
-                "${ENVOY_FILTER_EXAMPLE_TESTS[@]}"
-            popd
-        fi
         # TODO(mattklein123): This part of the test is now flaky in CI and it's unclear why, possibly
         # due to sandboxing issue. Debug and enable it again.
         # if [ "${CI_SKIP_INTEGRATION_TEST_TRAFFIC_TAPPING}" != "1" ] ; then
@@ -424,7 +421,6 @@ case $CI_TARGET in
         setup_clang_toolchain
         # This doesn't go into CI but is available for developer convenience.
         echo "bazel with different compiletime options build with tests..."
-        cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"
         TEST_TARGETS=("${TEST_TARGETS[@]/#\/\//@envoy\/\/}")
         # Building all the dependencies from scratch to link them against libc++.
         echo "Building and testing with wasm=wamr: ${TEST_TARGETS[*]}"
@@ -972,16 +968,6 @@ case $CI_TARGET in
              --build_tests_only \
              --remote_download_minimal \
              "${TEST_TARGETS[@]}"
-        if [ "${ENVOY_BUILD_FILTER_EXAMPLE}" == "1" ]; then
-            echo "Building and testing envoy-filter-example tests..."
-            pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}"
-            bazel_with_collection \
-                test "${BAZEL_BUILD_OPTIONS[@]}" \
-                -c dbg \
-                --config=clang-tsan \
-                "${ENVOY_FILTER_EXAMPLE_TESTS[@]}"
-            popd
-        fi
         ;;
 
     verify_distro)

ci/filter_example_setup.sh

@@ -29,6 +29,6 @@ cp -a "${ENVOY_SRCDIR}"/bazel/protoc "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/bazel/
 cp -f "${ENVOY_SRCDIR}"/.bazelrc "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/
 rm -f "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/.bazelversion
 cp -f "${ENVOY_SRCDIR}"/.bazelversion "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/
-cp -f "${ENVOY_SRCDIR}"/*.bazelrc "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/
+cp -f "${ENVOY_SRCDIR}"/*.bazelrc "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/ || :
 
 export FILTER_WORKSPACE_SET=1

contrib/client_ssl_auth/filters/network/source/client_ssl_auth.cc

@@ -35,7 +35,7 @@ ClientSslAuthConfig::ClientSslAuthConfig(
           std::chrono::milliseconds(1000)),
       tls_(tls.allocateSlot()), stats_(generateStats(scope, config.stat_prefix())) {
   auto list_or_error = Network::Address::IpList::create(config.ip_white_list());
-  THROW_IF_STATUS_NOT_OK(list_or_error, throw);
+  THROW_IF_NOT_OK_REF(list_or_error.status());
   ip_allowlist_ = std::move(list_or_error.value());
 
   if (!cm.clusters().hasCluster(remote_cluster_name_)) {

contrib/golang/filters/http/source/golang_filter.cc

@@ -19,6 +19,7 @@
 #include "source/common/grpc/status.h"
 #include "source/common/http/headers.h"
 #include "source/common/http/http1/codec_impl.h"
+#include "source/common/http/utility.h"
 #include "source/common/router/string_accessor_impl.h"
 #include "source/extensions/filters/common/expr/context.h"
 
@@ -1377,16 +1378,13 @@ uint64_t Filter::getMergedConfigId() {
   Http::StreamFilterCallbacks* callbacks = decoding_state_.getFilterCallbacks();
 
   // get all of the per route config
-  std::list<const FilterConfigPerRoute*> route_config_list;
-  callbacks->traversePerFilterConfig(
-      [&route_config_list](const Router::RouteSpecificFilterConfig& cfg) {
-        route_config_list.push_back(dynamic_cast<const FilterConfigPerRoute*>(&cfg));
-      });
+  auto route_config_list = Http::Utility::getAllPerFilterConfig<FilterConfigPerRoute>(callbacks);
 
   ENVOY_LOG(debug, "golang filter route config list length: {}.", route_config_list.size());
 
   auto id = config_->getConfigId();
   for (auto it : route_config_list) {
+    ASSERT(it != nullptr, "route config should not be null");
     auto route_config = *it;
     id = route_config.getPluginConfigId(id, config_->pluginName());
   }

contrib/sip_proxy/filters/network/source/tra/tra_impl.cc

@@ -204,7 +204,7 @@ ClientPtr traClient(Event::Dispatcher& dispatcher, Server::Configuration::Factor
                              .clusterManager()
                              .grpcAsyncClientManager()
                              .getOrCreateRawAsyncClient(grpc_service, context.scope(), true);
-  THROW_IF_STATUS_NOT_OK(client_or_error, throw);
+  THROW_IF_NOT_OK_REF(client_or_error.status());
   return std::make_unique<SipProxy::TrafficRoutingAssistant::GrpcClientImpl>(
       client_or_error.value(), dispatcher, timeout);
 }

docs/root/configuration/http/http_filters/original_src_filter.rst

@@ -70,7 +70,7 @@ The following example configures Envoy to use the original source for all connec
   http_filters:
     - name: envoy.filters.http.original_src
       typed_config:
-        "@type": type.googleapis.com/envoy.extensions.filters.listener.original_src.v3.OriginalSrc
+        "@type": type.googleapis.com/envoy.extensions.filters.http.original_src.v3.OriginalSrc
         mark: 123
     - name: envoy.filters.http.router
       typed_config: